Support for z/OS archive files (#904)

Co-authored-by: Philip Craig <philipjcraig@gmail.com>

src/archive.rs

@@ -19,6 +19,16 @@ pub const THIN_MAGIC: [u8; 8] = *b"!<thin>\n";
 /// The terminator for each archive member header.
 pub const TERMINATOR: [u8; 2] = *b"`\n";
 
+/// File identification bytes at the beginning of a z/OS archive.
+///
+/// This is [`MAGIC`] in EBCDIC.
+pub const ZOS_MAGIC: [u8; 8] = *b"\x5A\x4C\x81\x99\x83\x88\x6E\x15";
+
+/// The terminator for each z/OS archive member header.
+///
+/// This is [`TERMINATOR`] in EBCDIC.
+pub const ZOS_TERMINATOR: [u8; 2] = *b"\x79\x15";
+
 /// The header at the start of an archive member.
 #[derive(Debug, Clone, Copy)]
 #[repr(C)]

src/read/archive.rs

@@ -47,6 +47,8 @@ pub enum ArchiveKind {
     Coff,
     /// The AIX big archive format.
     AixBig,
+    /// The z/OS archive format.
+    Zos,
 }
 
 /// The list of members in the archive.
@@ -59,6 +61,10 @@ enum Members<'data> {
     AixBig {
         index: &'data [archive::AixMemberOffset],
     },
+    Zos {
+        offset: u64,
+        end_offset: u64,
+    },
 }
 
 /// A partially parsed archive file.
@@ -81,9 +87,13 @@ impl<'data, R: ReadRef<'data>> ArchiveFile<'data, R> {
             .read_bytes(&mut tail, archive::MAGIC.len() as u64)
             .read_error("Invalid archive size")?;
 
-        let thin = if magic == archive::AIX_BIG_MAGIC {
+        if magic == archive::AIX_BIG_MAGIC {
             return Self::parse_aixbig(data);
-        } else if magic == archive::THIN_MAGIC {
+        } else if magic == archive::ZOS_MAGIC {
+            return Self::parse_zos(data);
+        }
+
+        let thin = if magic == archive::THIN_MAGIC {
             true
         } else if magic == archive::MAGIC {
             false
@@ -265,6 +275,46 @@ impl<'data, R: ReadRef<'data>> ArchiveFile<'data, R> {
         Ok(file)
     }
 
+    /// Parse the z/OS archive header and special __.SYMDEF member.
+    fn parse_zos(data: R) -> read::Result<Self> {
+        let len = data.len().read_error("Unknown archive length")?;
+        let mut tail = 0;
+        let magic = data
+            .read_bytes(&mut tail, archive::MAGIC.len() as u64)
+            .read_error("Invalid z/OS archive size")?;
+        debug_assert_eq!(magic, archive::ZOS_MAGIC);
+
+        let mut file = ArchiveFile {
+            data: SkipDebugList(data),
+            kind: ArchiveKind::Zos,
+            members: Members::Zos {
+                offset: 0,
+                end_offset: 0,
+            },
+            symbols: (0, 0),
+            names: &[],
+            thin: false,
+        };
+
+        let mut members_offset = tail;
+        let members_end_offset = len;
+
+        if tail < len {
+            let member = ArchiveMember::parse_zos(data, &mut tail)?;
+            // __.SYMDEF
+            if member.name() == [0x6D, 0x6D, 0x4B, 0xE2, 0xE8, 0xD4, 0xC4, 0xC5, 0xC6] {
+                // Symbol table member.
+                file.symbols = member.file_range();
+                members_offset = tail;
+            }
+        }
+        file.members = Members::Zos {
+            offset: members_offset,
+            end_offset: members_end_offset,
+        };
+        Ok(file)
+    }
+
     /// Return the archive format.
     #[inline]
     pub fn kind(&self) -> ArchiveKind {
@@ -303,6 +353,13 @@ impl<'data, R: ReadRef<'data>> ArchiveFile<'data, R> {
                 let offset = member.0;
                 ArchiveMember::parse_aixbig(self.data.0, offset)
             }
+            Members::Zos { offset, end_offset } => {
+                if member.0 < offset || member.0 >= end_offset {
+                    return Err(Error("Invalid archive member offset"));
+                }
+                let mut offset = member.0;
+                ArchiveMember::parse_zos(self.data.0, &mut offset)
+            }
         }
     }
 
@@ -353,6 +410,16 @@ impl<'data, R: ReadRef<'data>> Iterator for ArchiveMemberIterator<'data, R> {
                     Some(member)
                 }
             },
+            Members::Zos { offset, end_offset } => {
+                if *offset >= *end_offset {
+                    return None;
+                }
+                let member = ArchiveMember::parse_zos(self.data, offset);
+                if member.is_err() {
+                    *offset = *end_offset;
+                }
+                Some(member)
+            }
         }
     }
 }
@@ -364,6 +431,9 @@ enum MemberHeader<'data> {
     Common(&'data archive::Header),
     /// AIX big archive header
     AixBig(&'data archive::AixHeader),
+    /// ZOS archive header.
+    /// Same format as Common but fields are EBCDIC.
+    Zos(&'data archive::Header),
 }
 
 /// A partially parsed archive member.
@@ -446,6 +516,50 @@ impl<'data> ArchiveMember<'data> {
         })
     }
 
+    /// Parse the member header, name, and file data in an archive with the z/OS format.
+    ///
+    /// This reads the extended name (if any) and adjusts the file size.
+    fn parse_zos<R: ReadRef<'data>>(data: R, offset: &mut u64) -> read::Result<Self> {
+        let header = data
+            .read::<archive::Header>(offset)
+            .read_error("Invalid z/OS archive member header")?;
+        if header.terminator != archive::ZOS_TERMINATOR {
+            return Err(Error("Invalid z/OS archive header terminator"));
+        }
+
+        let header_file_size =
+            parse_u64_ebcdic_digits(&header.size, 10).read_error("Invalid archive member size")?;
+        let mut file_offset = *offset;
+        let mut file_size = header_file_size;
+
+        // Check for "#1/" in EBCDIC followed by EBCDIC length.
+        let name = if header.name[..3] == [0x7B, 0xF1, 0x61] && is_ebcdic_digit(header.name[3]) {
+            // BSD-style extended name at start of file data.
+            // Note: This is unlikely for z/OS archives, but we handle it for completeness
+            zos_parse_bsd_extended_name(&header.name[3..], data, &mut file_offset, &mut file_size)
+                .read_error("Invalid archive extended name length")?
+        } else {
+            let name_len = memchr::memchr(EBCDIC_SPACE, &header.name).unwrap_or(header.name.len());
+            &header.name[..name_len]
+        };
+
+        // Skip the file data.
+        *offset = offset
+            .checked_add(header_file_size)
+            .read_error("Archive member size is too large")?;
+        // Entries are padded to an even number of bytes.
+        if (header_file_size & 1) != 0 {
+            *offset = offset.saturating_add(1);
+        }
+
+        Ok(ArchiveMember {
+            header: MemberHeader::Zos(header),
+            name,
+            offset: file_offset,
+            size: file_size,
+        })
+    }
+
     /// Parse a member index entry in an AIX big archive,
     /// and then parse the member header, name, and file data.
     fn parse_aixbig_index<R: ReadRef<'data>>(
@@ -530,6 +644,7 @@ impl<'data> ArchiveMember<'data> {
         match &self.header {
             MemberHeader::Common(header) => parse_u64_digits(&header.date, 10),
             MemberHeader::AixBig(header) => parse_u64_digits(&header.date, 10),
+            MemberHeader::Zos(header) => parse_u64_ebcdic_digits(&header.date, 10),
         }
     }
 
@@ -539,6 +654,7 @@ impl<'data> ArchiveMember<'data> {
         match &self.header {
             MemberHeader::Common(header) => parse_u64_digits(&header.uid, 10),
             MemberHeader::AixBig(header) => parse_u64_digits(&header.uid, 10),
+            MemberHeader::Zos(header) => parse_u64_ebcdic_digits(&header.uid, 10),
         }
     }
 
@@ -548,6 +664,7 @@ impl<'data> ArchiveMember<'data> {
         match &self.header {
             MemberHeader::Common(header) => parse_u64_digits(&header.gid, 10),
             MemberHeader::AixBig(header) => parse_u64_digits(&header.gid, 10),
+            MemberHeader::Zos(header) => parse_u64_ebcdic_digits(&header.gid, 10),
         }
     }
 
@@ -557,6 +674,7 @@ impl<'data> ArchiveMember<'data> {
         match &self.header {
             MemberHeader::Common(header) => parse_u64_digits(&header.mode, 8),
             MemberHeader::AixBig(header) => parse_u64_digits(&header.mode, 8),
+            MemberHeader::Zos(header) => parse_u64_ebcdic_digits(&header.mode, 8),
         }
     }
 
@@ -720,6 +838,8 @@ impl<'data> ArchiveSymbolIterator<'data> {
             }
             // TODO: Implement AIX big archive symbol table.
             ArchiveKind::AixBig => Ok(ArchiveSymbolIterator(SymbolIteratorInternal::None)),
+            // TODO: Implement z/OS archive symbol table.
+            ArchiveKind::Zos => Ok(ArchiveSymbolIterator(SymbolIteratorInternal::None)),
         }
     }
 }
@@ -857,6 +977,39 @@ fn parse_u64_digits(digits: &[u8], radix: u32) -> Option<u64> {
     Some(result)
 }
 
+/// Parse EBCDIC-encoded numeric digits.
+fn parse_u64_ebcdic_digits(digits: &[u8], radix: u32) -> Option<u64> {
+    if let [EBCDIC_SPACE, ..] = digits {
+        return None;
+    }
+    let mut result: u64 = 0;
+    for &c in digits {
+        if c == EBCDIC_SPACE {
+            return Some(result);
+        } else {
+            let x = ebcdic_to_digit(c, radix)?;
+            result = result
+                .checked_mul(u64::from(radix))?
+                .checked_add(u64::from(x))?;
+        }
+    }
+    Some(result)
+}
+
+const EBCDIC_SPACE: u8 = 0x40;
+
+fn is_ebcdic_digit(byte: u8) -> bool {
+    matches!(byte, 0xF0..=0xF9)
+}
+
+fn ebcdic_to_digit(byte: u8, radix: u32) -> Option<u8> {
+    if radix == 8 && matches!(byte, 0xF0..=0xF7) || radix == 10 && matches!(byte, 0xF0..=0xF9) {
+        Some(byte - 0xF0)
+    } else {
+        None
+    }
+}
+
 /// Digits are a decimal offset into the extended name table.
 /// Name is terminated by "/\n" (for GNU) or a null byte (for COFF).
 fn parse_sysv_extended_name<'data>(digits: &[u8], names: &'data [u8]) -> Result<&'data [u8], ()> {
@@ -894,6 +1047,25 @@ fn parse_bsd_extended_name<'data, R: ReadRef<'data>>(
     Ok(name)
 }
 
+/// Digits are an EBCDIC decimal length of the extended name, which is contained
+/// in `data` at `offset`.
+/// Modifies `offset` and `size` to start after the extended name.
+fn zos_parse_bsd_extended_name<'data, R: ReadRef<'data>>(
+    digits: &[u8],
+    data: R,
+    offset: &mut u64,
+    size: &mut u64,
+) -> Result<&'data [u8], ()> {
+    let len = parse_u64_ebcdic_digits(digits, 10).ok_or(())?;
+    *size = size.checked_sub(len).ok_or(())?;
+    let name_data = data.read_bytes(offset, len)?;
+    let name = match memchr::memchr(b'\0', name_data) {
+        Some(len) => &name_data[..len],
+        None => name_data,
+    };
+    Ok(name)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1141,4 +1313,43 @@ mod tests {
 
         assert!(members.next().is_none());
     }
+
+    #[test]
+    fn zos_names() {
+        let data = b"\
+            \x5A\x4C\x81\x99\x83\x88\x6E\x15\
+            \xA3\x85\xA2\xA3\xF1\x4B\x96\x40\x40\x40\x40\x40\x40\x40\x40\x40\
+            \xF1\xF6\xF6\xF2\xF6\xF1\xF0\xF3\xF7\xF0\x40\x40\
+            \xF2\xF2\xF3\x40\x40\x40\
+            \xF1\x40\x40\x40\x40\x40\
+            \xF6\xF4\xF4\x40\x40\x40\x40\x40\
+            \xF6\x40\x40\x40\x40\x40\x40\x40\x40\x40\
+            \x79\x15\
+            first\0\
+            \xA3\x85\xA2\xA3\xF2\x4B\x96\x40\x40\x40\x40\x40\x40\x40\x40\x40\
+            \xF1\xF6\xF6\xF2\xF6\xF1\xF0\xF3\xF7\xF0\x40\x40\
+            \xF2\xF2\xF3\x40\x40\x40\
+            \xF1\x40\x40\x40\x40\x40\
+            \xF6\xF4\xF4\x40\x40\x40\x40\x40\
+            \xF7\x40\x40\x40\x40\x40\x40\x40\x40\x40\
+            \x79\x15\
+            second\0";
+        let data = &data[..];
+        let archive = ArchiveFile::parse(data).unwrap();
+        assert_eq!(archive.kind(), ArchiveKind::Zos);
+        let mut members = archive.members();
+
+        let member = members.next().unwrap().unwrap();
+        // Names are returned in raw EBCDIC encoding
+        // "test1.o" in EBCDIC = \xA3\x85\xA2\xA3\xF1\x4B\x96
+        assert_eq!(member.name(), b"\xA3\x85\xA2\xA3\xF1\x4B\x96");
+        assert_eq!(member.data(data).unwrap(), &b"first\0"[..]);
+
+        let member = members.next().unwrap().unwrap();
+        // "test2.o" in EBCDIC = \xA3\x85\xA2\xA3\xF2\x4B\x96
+        assert_eq!(member.name(), b"\xA3\x85\xA2\xA3\xF2\x4B\x96");
+        assert_eq!(member.data(data).unwrap(), &b"second\0"[..]);
+
+        assert!(members.next().is_none());
+    }
 }