Fix SLF string scanning and log decoding

Author: fortmarek

Package.swift

@@ -38,7 +38,10 @@ let package = Package(
         ),
         .testTarget(
             name: "XCLogParserTests",
-            dependencies: ["XCLogParser"]
+            dependencies: [
+                "XCLogParser",
+                .product(name: "Gzip", package: "GzipSwift")
+            ]
         ),
     ]
 

Package@swift-4.2.swift

@@ -31,7 +31,7 @@ let package = Package(
         ),
         .testTarget(
             name: "XCLogParserTests",
-            dependencies: ["XCLogParser"]
+            dependencies: ["XCLogParser", "Gzip"]
         ),
     ]
 

Package@swift-5.0.swift

@@ -34,7 +34,7 @@ let package = Package(
         ),
         .testTarget(
             name: "XCLogParserTests",
-            dependencies: ["XCLogParser"]
+            dependencies: ["XCLogParser", "Gzip"]
         ),
     ]
 

Sources/XCLogParser/lexer/Scanner.swift

@@ -44,8 +44,6 @@ final class Scanner {
         let end = String.Index(compilerSafeOffset: endOffset, in: self.string)
         let result = self.string[start..<end]
 
-        guard result.count == count else { return nil }
-
         self.offset += count
 
         return String(result)

Sources/XCLogParser/loglocation/LogLoader.swift

@@ -26,17 +26,7 @@ public struct LogLoader {
         do {
             let data = try Data(contentsOf: url)
             let unzipped = try data.gunzipped()
-            let string: String? = unzipped.withUnsafeBytes { pointer in
-                guard let charPointer = pointer
-                    .assumingMemoryBound(to: CChar.self)
-                    .baseAddress
-                else {
-                    return nil
-                }
-
-                return String(cString: charPointer, encoding: .ascii)
-            }
-            guard let contents = string else {
+            guard let contents = String(data: unzipped, encoding: .utf8) else {
                 throw LogError.readingFile(url.path)
             }
             return contents

Tests/XCLogParserTests/ActivityParserTests.swift

@@ -17,6 +17,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import Foundation
+import Gzip
 import XCTest
 @testable import XCLogParser
 
@@ -584,6 +586,30 @@ class ActivityParserTests: XCTestCase {
         XCTAssertEqual(10, parser.logVersion)
     }
 
+    func testParseActivityLogInURLPreservesUTF16LengthStringsAndEmbeddedNullBytes() throws {
+        let title = "Sources/Bundle+Locali🙂\u{0}zation"
+        var sectionTokens = IDEActivityLogSectionTokensWithoutAttachments
+        sectionTokens[2] = Token.string(title)
+        let tokens = [
+            Token.int(10),
+            Token.className("IDECommandLineBuildLog"),
+            Token.classNameRef("IDECommandLineBuildLog")
+        ] + sectionTokens
+        let directory = try TestUtils.createRandomTestDir()
+        defer { try? FileManager.default.removeItem(at: directory) }
+        let logURL = try writeActivityLog(contents: try slfContents(from: tokens), in: directory)
+
+        let activityLog = try parser.parseActivityLogInURL(
+            logURL,
+            redacted: false,
+            withoutBuildSpecificInformation: false
+        )
+
+        XCTAssertEqual(10, activityLog.version)
+        XCTAssertEqual(title, activityLog.mainSection.title)
+        XCTAssertEqual(10, parser.logVersion)
+    }
+
     func testParseDBGConsoleLog() throws {
         parser.logVersion = 11
         let tokens = DBGConsoleLogTokens
@@ -714,4 +740,42 @@ class ActivityParserTests: XCTestCase {
         XCTAssertEqual(metrics.taskCounters["SwiftDriver"]?["x"], 2)
     }
 
+    private func writeActivityLog(contents: String, in directory: URL) throws -> URL {
+        let url = directory.appendingPathComponent("test.xcactivitylog")
+        let data = contents.data(using: .utf8)!
+        try data.gzipped().write(to: url)
+        return url
+    }
+
+    private func slfContents(from tokens: [Token]) throws -> String {
+        var classNames: [String] = []
+        var contents = "SLF"
+        for token in tokens {
+            switch token {
+            case .int(let value):
+                contents += "\(value)#"
+            case .className(let value):
+                classNames.append(value)
+                contents += "\(value.utf16.count)%\(value)"
+            case .classNameRef(let value):
+                let index = try XCTUnwrap(
+                    classNames.firstIndex(of: value),
+                    "Class name \(value) must be emitted before it is referenced"
+                )
+                contents += "\(index + 1)@"
+            case .string(let value):
+                contents += "\(value.utf16.count)\"\(value)"
+            case .double(let value):
+                contents += "\(String(value.bitPattern.byteSwapped, radix: 16))^"
+            case .null:
+                contents += "-"
+            case .list(let value):
+                contents += "\(value)("
+            case .json(let value):
+                contents += "\(value.utf16.count)*\(value)"
+            }
+        }
+        return contents
+    }
+
 }

Tests/XCLogParserTests/LexerTests.swift

@@ -56,6 +56,15 @@ class LexerTests: XCTestCase {
         XCTAssertEqual(stringToken, Token.string("Xcode.IDEActivityLogDomainType.BuildLog"))
     }
 
+    func testTokenizeStringUsesUTF16Length() throws {
+        let value = "Sources/Bundle+Locali🙂"
+        let logContents = "SLF0#\(value.utf16.count)\"\(value)1#"
+
+        let tokens = try lexer.tokenize(contents: logContents, redacted: false, withoutBuildSpecificInformation: false)
+
+        XCTAssertEqual(tokens, [.int(0), .string(value), .int(1)])
+    }
+
     func testTokenizeDouble() throws {
         let logContents = "SLF09#21%IDEActivityLogSection1@39\"Xcode.IDEActivityLogDomainType.BuildLog356098f239dfc041^"
         let tokens = try lexer.tokenize(contents: logContents, redacted: false, withoutBuildSpecificInformation: false)

Tests/XCLogParserTests/LogLoaderTests.swift

@@ -0,0 +1,40 @@
+// Copyright (c) 2019 Spotify AB.
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+import Gzip
+import XCTest
+@testable import XCLogParser
+
+class LogLoaderTests: XCTestCase {
+
+    func testLoadFromURLPreservesEmbeddedNullBytes() throws {
+        let directory = try TestUtils.createRandomTestDir()
+        defer { try? FileManager.default.removeItem(at: directory) }
+        let url = directory.appendingPathComponent("test.xcactivitylog")
+        let value = "Sources/Bundle+Locali\u{0}zation"
+        let contents = "SLF0#\(value.utf16.count)\"\(value)1#"
+        let data = contents.data(using: .utf8)!
+        try data.gzipped().write(to: url)
+
+        let loadedContents = try LogLoader().loadFromURL(url)
+
+        XCTAssertEqual(loadedContents, contents)
+    }
+}

Tests/XCLogParserTests/XCTestManifests.swift

@@ -7,6 +7,7 @@ extension ActivityParserTests {
     // to regenerate.
     static let __allTests__ActivityParserTests = [
         ("testParseActivityLog", testParseActivityLog),
+        ("testParseActivityLogInURLPreservesUTF16LengthStringsAndEmbeddedNullBytes", testParseActivityLogInURLPreservesUTF16LengthStringsAndEmbeddedNullBytes),
         ("testParseDBGConsoleLog", testParseDBGConsoleLog),
         ("testParseDVTTextDocumentLocation", testParseDVTTextDocumentLocation),
         ("testParseIBDocumentMemberLocation", testParseIBDocumentMemberLocation),
@@ -65,10 +66,20 @@ extension LexerTests {
         ("testTokenizeListNil", testTokenizeListNil),
         ("testTokenizeString", testTokenizeString),
         ("testTokenizeStringRedacted", testTokenizeStringRedacted),
+        ("testTokenizeStringUsesUTF16Length", testTokenizeStringUsesUTF16Length),
         ("testTokenizeStringWithTokenDelimiters", testTokenizeStringWithTokenDelimiters),
     ]
 }
 
+extension LogLoaderTests {
+    // DO NOT MODIFY: This is autogenerated, use:
+    //   `swift test --generate-linuxmain`
+    // to regenerate.
+    static let __allTests__LogLoaderTests = [
+        ("testLoadFromURLPreservesEmbeddedNullBytes", testLoadFromURLPreservesEmbeddedNullBytes),
+    ]
+}
+
 extension LogFinderTests {
     // DO NOT MODIFY: This is autogenerated, use:
     //   `swift test --generate-linuxmain`
@@ -138,6 +149,7 @@ public func __allTests() -> [XCTestCaseEntry] {
         testCase(IssuesReporterTests.__allTests__IssuesReporterTests),
         testCase(LexRedactorTests.__allTests__LexRedactorTests),
         testCase(LexerTests.__allTests__LexerTests),
+        testCase(LogLoaderTests.__allTests__LogLoaderTests),
         testCase(LogFinderTests.__allTests__LogFinderTests),
         testCase(LogManifestTests.__allTests__LogManifestTests),
         testCase(ParserTests.__allTests__ParserTests),