SAP IDoc FlatFile to XML converter

One of our projects is to create/change hundreds of maps and switch from SAP IDoc flat files to XML files. All of the test files are in the FlatFile format, so I decided to create a converter to translate them into XML. It’s clear that this task requires some kind of a library with the IDoc documents structures description (hierarchy, records, positions, etc). I decided to use IDoc DDFs (format used by IBM/Sterling Map Editor for creating structures – you can download them from GIS/SI or just use existing GIS/SI map) and XSLT for it, and several hours later can convert my test data from FF to XML in seconds:

Just an idea.

PS I don’t have access to SAP at the moment so I cannot use it for XML Idoc generation. Also, I have appropriate I/Os for existing IDocs so it made sense to convert these existing files.

PPS it’s clear that an XML->FF converter is easier to develop – you don’t need to care about the hierarchy.

Here is my XSLT code. It doesn’t work without appropriate DDF file so if you have no idea about XSL/DDF you won’t be able to use it. It’s for those who is experienced but lazy – just like me 🙂

<?xml version="1.0"?>
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
	<xsl:output indent="yes"/>
	<xsl:variable name="nl">
		<xsl:text>
</xsl:text>
	</xsl:variable>

	<!--xsl:variable name="process_empty_elts" select="'yes'" /-->
	<xsl:variable name="process_empty_elts" select="'no'" />

	<xsl:variable name="max_len" select="20" />

	<xsl:template match="/">
		<xsl:variable name="data" select="/*/text()" />
		<xsl:variable name="main_type" select="normalize-space(substring($data, 40, 15))" />

		<xsl:variable name="type">
			<xsl:choose>
				<xsl:when test="normalize-space(substring($data, 70, 15)) != ''">
					<xsl:value-of select="normalize-space(substring($data, 70, 15))" />
				</xsl:when>
				<xsl:otherwise>
					<xsl:value-of select="$main_type"/>
				</xsl:otherwise>
			</xsl:choose>
		</xsl:variable>

		<xsl:variable name="structure" select="concat($main_type, '.xml')"/>

		<xsl:variable name="top_siblings">
			<xsl:for-each select="document($structure)/*/*/*/@NAME">
				<xsl:value-of select="."/>
				<xsl:value-of select="'~'"/>
			</xsl:for-each>
		</xsl:variable>

		<xsl:variable name="all_children">
			<xsl:for-each select="document($structure)//*/@NAME">
				<xsl:value-of select="."/>
				<xsl:value-of select="'~'"/>
			</xsl:for-each>
		</xsl:variable>

		<xsl:element name="{$type}">
			<IDOC BEGIN="1">
				<xsl:call-template name="process_line">
					<xsl:with-param name="data" select="$data" />
					<xsl:with-param name="structure" select="$structure" />
					<xsl:with-param name="siblings" select="$top_siblings" />
					<xsl:with-param name="children" select="$all_children" />
				</xsl:call-template>
			</IDOC>
		</xsl:element>
	</xsl:template>

	<!--
		=== line by line ====================================
	-->
	<xsl:template name="process_line">
		<xsl:param name="data" select="''" />
		<xsl:param name="structure" />
		<xsl:param name="siblings" />
		<xsl:param name="children" />

		<xsl:choose>
			<xsl:when test="contains($data, $nl)">
				<!-- not the last record -->
				<xsl:variable name="record" select="substring-before($data, $nl)" />
				<xsl:variable name="record_name_tmp">
					<xsl:choose>
						<xsl:when test="starts-with($record, 'EDI_DC40')">
							<xsl:value-of select="'EDI_DC40'"/>
						</xsl:when>
						<xsl:when test="starts-with($record, 'Z')">
							<xsl:choose>
								<!-- Z-custom, check first 9 symbols -->
								<xsl:when test="count(document($structure)//POSREC[contains(@NAME, substring($record, 1, 9))]) != 0">
									<xsl:choose>
										<xsl:when test="contains(document($structure)//POSREC[contains(@NAME, substring($record, 1, 9))]/@NAME, ':')">
											<xsl:value-of select="substring-before(document($structure)//POSREC[contains(@NAME, substring($record, 1, 9))]/@NAME, ':')"/>
										</xsl:when>
										<xsl:otherwise>
											<xsl:value-of select="document($structure)//POSREC[contains(@NAME, substring($record, 1, 9))]/@NAME"/>
										</xsl:otherwise>
									</xsl:choose>
								</xsl:when>
								<!-- Z-custom, then 8 symbols -->
								<xsl:when test="count(document($structure)//POSREC[contains(@NAME, substring($record, 1, 8))]) != 0">
									<xsl:choose>
										<xsl:when test="contains(document($structure)//POSREC[contains(@NAME, substring($record, 1, 8))]/@NAME, ':')">
											<xsl:value-of select="substring-before(document($structure)//POSREC[contains(@NAME, substring($record, 1, 8))]/@NAME, ':')"/>
										</xsl:when>
										<xsl:otherwise>
											<xsl:value-of select="document($structure)//POSREC[contains(@NAME, substring($record, 1, 8))]/@NAME"/>
										</xsl:otherwise>
									</xsl:choose>
								</xsl:when>
								<!-- Z-custom, then 7 symbols -->
								<xsl:when test="count(document($structure)//POSREC[contains(@NAME, substring($record, 1, 7))]) != 0">
									<xsl:choose>
										<xsl:when test="contains(document($structure)//POSREC[contains(@NAME, substring($record, 1, 7))]/@NAME, ':')">
											<xsl:value-of select="substring-before(document($structure)//POSREC[contains(@NAME, substring($record, 1, 7))]/@NAME, ':')"/>
										</xsl:when>
										<xsl:otherwise>
											<xsl:value-of select="document($structure)//POSREC[contains(@NAME, substring($record, 1, 7))]/@NAME"/>
										</xsl:otherwise>
									</xsl:choose>
								</xsl:when>
								<xsl:otherwise>
									<xsl:value-of select="substring($record, 1, 7)" />
								</xsl:otherwise>
							</xsl:choose>
						</xsl:when>
						<xsl:otherwise>
							<xsl:value-of select="substring($record, 1, 7)" />
						</xsl:otherwise>
					</xsl:choose>
				</xsl:variable>

				<!-- E2*, E1* - E1* -->
				<xsl:variable name="record_name">
					<xsl:choose>
						<xsl:when test="starts-with($record_name_tmp, 'E2')">
							<xsl:value-of select="concat( 'E1', substring-after($record_name_tmp, 'E2') )"/>
						</xsl:when>
						<xsl:otherwise>
							<xsl:value-of select="$record_name_tmp"/>
						</xsl:otherwise>
					</xsl:choose>
				</xsl:variable>

				<!--
					only if
					it's a sibling of the current level
					-or-
					this element is not in the schema
									==<xsl:value-of select="$record_name_tmp"/>~<xsl:value-of select="$siblings"/>
				-->
				<xsl:if test="contains($siblings, $record_name_tmp) or count(document($structure)//POSREC[contains(@NAME, $record_name_tmp)]) = 0">
					<xsl:if test="normalize-space($record_name) != ''">

						<xsl:if test="count(document($structure)//POSREC[contains(@NAME, $record_name_tmp)]) = 0">
							<xsl:comment>
								<xsl:value-of select="'========= !!! actual position of the record is unknown !!! ========='"/>
							</xsl:comment>
						</xsl:if>
						<xsl:element name="{$record_name}">
							<xsl:attribute name="SEGMENT">1</xsl:attribute>
							<xsl:call-template name="build_elements">
								<xsl:with-param name="record" select="$record" />
								<xsl:with-param name="record_name" select="$record_name_tmp" />
								<xsl:with-param name="structure" select="$structure" />
							</xsl:call-template>

							<!-- child elements -->
							<xsl:variable name="child_siblings">
								<xsl:for-each select="document($structure)//*[contains(@NAME, $record_name_tmp)]/*[not(contains(@NAME, $record_name_tmp))]/@NAME">
									<xsl:value-of select="."/>
									<xsl:value-of select="'~'"/>
								</xsl:for-each>
							</xsl:variable>

							<xsl:variable name="child_children">
								<xsl:for-each select="document($structure)//*[contains(@NAME, $record_name_tmp)]//*[not(contains(@NAME, $record_name_tmp))]/@NAME">
									<xsl:value-of select="."/>
									<xsl:value-of select="'~'"/>
								</xsl:for-each>
							</xsl:variable>

							<xsl:if test="count(document($structure)//POSREC[contains(@NAME, $record_name_tmp)]) != 0">
								<xsl:call-template name="process_line">
									<xsl:with-param name="data" select="substring-after($data, $nl)" />
									<xsl:with-param name="structure" select="$structure" />
									<xsl:with-param name="siblings" select="$child_siblings" />
									<xsl:with-param name="children" select="$child_children" />
								</xsl:call-template>
							</xsl:if>
						</xsl:element>
					</xsl:if>
				</xsl:if>

				<!-- process the rest -->
				<xsl:if test="contains($children, $record_name_tmp) or count(document($structure)//POSREC[contains(@NAME, $record_name_tmp)]) = 0">
					<xsl:call-template name="process_line">
						<xsl:with-param name="data" select="substring-after($data, $nl)" />
						<xsl:with-param name="structure" select="$structure" />
						<xsl:with-param name="siblings" select="$siblings" />
						<xsl:with-param name="children" select="$children" />
					</xsl:call-template>
				</xsl:if>
			</xsl:when>
			<xsl:otherwise>

			</xsl:otherwise>
		</xsl:choose>
	</xsl:template>

	<!--
		=== elements =========================
	-->
	<xsl:template name="build_elements">
		<xsl:param name="record" select="''" />
		<xsl:param name="record_name" select="''" />
		<xsl:param name="structure" />

		<xsl:choose>
			<xsl:when test="count(document($structure)//POSREC[contains(@NAME, $record_name)]) != 0">

				<xsl:if test="$record_name = 'EDI_DC40'">
					<TABNAM>EDI_DC40</TABNAM>
				</xsl:if>

				<xsl:for-each select="(document($structure)//POSREC[contains(@NAME, $record_name)])[1]/POSFIELD">
					<xsl:if test="position() &gt; 6 or $record_name = 'EDI_DC40'">
						<xsl:variable name="field_name">
							<xsl:choose>
								<xsl:when test="contains(@NAME, ':')">
									<xsl:value-of select="substring-before(@NAME, ':')"/>
								</xsl:when>
								<xsl:otherwise>
									<xsl:value-of select="@NAME"/>
								</xsl:otherwise>
							</xsl:choose>
						</xsl:variable>

						<xsl:variable name="STARTPOS" select="@STARTPOS" />
						<xsl:variable name="LENGTH" select="@LENGTH" />

						<xsl:if test="$process_empty_elts = 'yes' or normalize-space( substring($record, (number($STARTPOS)+1), $LENGTH) ) != ''">
							<xsl:element name="{$field_name}">
								<xsl:value-of select="normalize-space( substring($record, (number($STARTPOS)+1), $LENGTH) )"/>
							</xsl:element>
						</xsl:if>
					</xsl:if>
				</xsl:for-each>
			</xsl:when>
			<xsl:otherwise>
				<SDATA>
					<data>
						<xsl:value-of select="substring($record, 64)" />
					</data>
					<!--
					<xsl:value-of select="substring($record, 64)" />
					-->
					<xsl:call-template name="SDATA">
						<xsl:with-param name="s" select="substring($record, 64)" />
					</xsl:call-template>
				</SDATA>
			</xsl:otherwise>
		</xsl:choose>
	</xsl:template>

	<xsl:template name="SDATA">
		<xsl:param name="s" select="''" />
		<xsl:param name="start" select="1" />

		<xsl:if test="normalize-space($s) != ''">
			<xsl:choose>
				<xsl:when test="string-length($s) &gt; $max_len">
					<data>
						<xsl:attribute name="range">
							<xsl:value-of select="$start"/>
							<xsl:value-of select="' - '"/>
							<xsl:value-of select="$start + $max_len - 1"/>
						</xsl:attribute>
						<xsl:value-of select="substring($s, 1, $max_len)"/>
					</data>

					<xsl:call-template name="SDATA">
						<xsl:with-param name="s" select="substring($s, $max_len + 1)" />
						<xsl:with-param name="start" select="$start + $max_len" />
					</xsl:call-template>
				</xsl:when>
				<xsl:otherwise>
					<data>
						<xsl:attribute name="range">
							<xsl:value-of select="$start"/>
							<xsl:value-of select="' - '"/>
							<xsl:value-of select="$start + $max_len"/>
						</xsl:attribute>
						<xsl:value-of select="$s"/>
					</data>
				</xsl:otherwise>
			</xsl:choose>
		</xsl:if>
	</xsl:template>
</xsl:stylesheet>

Gennady Kim

Advertisements

2 thoughts on “SAP IDoc FlatFile to XML converter

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s