Skip to content

Commit a3ffa16

Browse files
fortmarekfortmarek
authored
Fix SLF string scanning and log decoding (#254)
Signed-off-by: fortmarek <marekfort@me.com>
1 parent 0b908a0 commit a3ffa16

10 files changed

Lines changed: 144 additions & 46 deletions

File tree

‎Package.swift‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,10 @@ let package = Package(
3838
),
3939
.testTarget(
4040
name: "XCLogParserTests",
41-
dependencies: ["XCLogParser"]
41+
dependencies: [
42+
"XCLogParser",
43+
.product(name: "Gzip", package: "GzipSwift")
44+
]
4245
),
4346
]
4447

‎Package@swift-4.2.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ let package = Package(
3131
),
3232
.testTarget(
3333
name: "XCLogParserTests",
34-
dependencies: ["XCLogParser"]
34+
dependencies: ["XCLogParser", "Gzip"]
3535
),
3636
]
3737

‎Package@swift-5.0.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ let package = Package(
3434
),
3535
.testTarget(
3636
name: "XCLogParserTests",
37-
dependencies: ["XCLogParser"]
37+
dependencies: ["XCLogParser", "Gzip"]
3838
),
3939
]
4040

‎Sources/XCLogParser/lexer/Lexer.swift‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -264,16 +264,6 @@ public final class Lexer {
264264

265265
private extension Scanner {
266266
var approximateLine: String {
267-
let currentLocation = self.offset
268-
let contentSize = self.string.count
269-
270-
let start = String.Index(compilerSafeOffset: currentLocation, in: self.string)
271-
let endCount = contentSize - currentLocation > 21 ? currentLocation + 21 : contentSize - currentLocation
272-
let end = String.Index(compilerSafeOffset: endCount, in: self.string)
273-
274-
if end <= start {
275-
return String(self.string[start..<self.stringEndIndex])
276-
}
277-
return String(self.string[start..<end])
267+
preview(count: 21)
278268
}
279269
}

‎Sources/XCLogParser/lexer/Scanner.swift‎

Lines changed: 30 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -22,62 +22,71 @@ import Foundation
2222
final class Scanner {
2323

2424
let string: String
25+
private let bytes: [UInt8]
2526

2627
private(set) var offset: Int
27-
private(set) lazy var stringEndIndex: String.Index = self.string.endIndex
2828

2929
var isAtEnd: Bool {
30-
String.Index(compilerSafeOffset: self.offset, in: self.string) >= self.stringEndIndex
30+
offset >= bytes.count
3131
}
3232

3333
init(string: String) {
3434
self.string = string
35+
self.bytes = Array(string.utf8)
3536
self.offset = 0
3637
}
3738

3839
func scan(count: Int) -> String? {
39-
let start = String.Index(compilerSafeOffset: self.offset, in: self.string)
4040
let endOffset = self.offset + count
4141

42-
guard endOffset <= self.string.utf16.count else { return nil }
43-
44-
let end = String.Index(compilerSafeOffset: endOffset, in: self.string)
45-
let result = self.string[start..<end]
46-
47-
guard result.count == count else { return nil }
42+
guard count >= 0,
43+
endOffset <= bytes.count,
44+
let result = String(bytes: bytes[offset..<endOffset], encoding: .utf8)
45+
else { return nil }
4846

4947
self.offset += count
5048

51-
return String(result)
49+
return result
5250
}
5351

5452
func scan(string value: String) -> Bool {
55-
guard self.string.starts(with: value) else { return false }
53+
let valueBytes = Array(value.utf8)
54+
let endOffset = offset + valueBytes.count
55+
guard endOffset <= bytes.count,
56+
bytes[offset..<endOffset].elementsEqual(valueBytes)
57+
else { return false }
5658

57-
self.offset += value.count
59+
self.offset += valueBytes.count
5860
return true
5961
}
6062

6163
func scanCharacters(from allowedCharacters: Set<Character>) -> String? {
62-
var prefix: String = ""
63-
var characterIndex = String.Index(compilerSafeOffset: self.offset, in: self.string)
64-
65-
while characterIndex < self.stringEndIndex {
66-
let character = self.string[characterIndex]
64+
let allowedBytes = Set(
65+
allowedCharacters.compactMap { character -> UInt8? in
66+
let characterBytes = Array(String(character).utf8)
67+
return characterBytes.count == 1 ? characterBytes[0] : nil
68+
}
69+
)
70+
var prefix: [UInt8] = []
6771

68-
guard allowedCharacters.contains(character) else {
72+
while !isAtEnd {
73+
guard allowedBytes.contains(bytes[offset]) else {
6974
break
7075
}
7176

72-
prefix.append(character)
77+
prefix.append(bytes[offset])
7378
self.offset += 1
74-
characterIndex = String.Index(utf16Offset: self.offset, in: self.string)
7579
}
7680

77-
return prefix
81+
return String(bytes: prefix, encoding: .utf8)
7882
}
7983

8084
func moveOffset(by value: Int) {
8185
self.offset += value
8286
}
87+
88+
func preview(count: Int) -> String {
89+
let endOffset = min(offset + count, bytes.count)
90+
return String(decoding: bytes[offset..<endOffset], as: UTF8.self)
91+
}
8392
}

‎Sources/XCLogParser/loglocation/LogLoader.swift‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,17 +26,7 @@ public struct LogLoader {
2626
do {
2727
let data = try Data(contentsOf: url)
2828
let unzipped = try data.gunzipped()
29-
let string: String? = unzipped.withUnsafeBytes { pointer in
30-
guard let charPointer = pointer
31-
.assumingMemoryBound(to: CChar.self)
32-
.baseAddress
33-
else {
34-
return nil
35-
}
36-
37-
return String(cString: charPointer, encoding: .ascii)
38-
}
39-
guard let contents = string else {
29+
guard let contents = String(data: unzipped, encoding: .utf8) else {
4030
throw LogError.readingFile(url.path)
4131
}
4232
return contents

‎Tests/XCLogParserTests/ActivityParserTests.swift‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
// specific language governing permissions and limitations
1818
// under the License.
1919

20+
import Foundation
21+
import Gzip
2022
import XCTest
2123
@testable import XCLogParser
2224

@@ -584,6 +586,30 @@ class ActivityParserTests: XCTestCase {
584586
XCTAssertEqual(10, parser.logVersion)
585587
}
586588

589+
func testParseActivityLogInURLPreservesUTF8ByteLengthStringsAndEmbeddedNullBytes() throws {
590+
let title = "➜ Sources/Bundle+Locali🙂\u{0}zation"
591+
var sectionTokens = IDEActivityLogSectionTokensWithoutAttachments
592+
sectionTokens[2] = Token.string(title)
593+
let tokens = [
594+
Token.int(10),
595+
Token.className("IDECommandLineBuildLog"),
596+
Token.classNameRef("IDECommandLineBuildLog")
597+
] + sectionTokens
598+
let directory = try TestUtils.createRandomTestDir()
599+
defer { try? FileManager.default.removeItem(at: directory) }
600+
let logURL = try writeActivityLog(contents: try slfContents(from: tokens), in: directory)
601+
602+
let activityLog = try parser.parseActivityLogInURL(
603+
logURL,
604+
redacted: false,
605+
withoutBuildSpecificInformation: false
606+
)
607+
608+
XCTAssertEqual(10, activityLog.version)
609+
XCTAssertEqual(title, activityLog.mainSection.title)
610+
XCTAssertEqual(10, parser.logVersion)
611+
}
612+
587613
func testParseDBGConsoleLog() throws {
588614
parser.logVersion = 11
589615
let tokens = DBGConsoleLogTokens
@@ -714,4 +740,42 @@ class ActivityParserTests: XCTestCase {
714740
XCTAssertEqual(metrics.taskCounters["SwiftDriver"]?["x"], 2)
715741
}
716742

743+
private func writeActivityLog(contents: String, in directory: URL) throws -> URL {
744+
let url = directory.appendingPathComponent("test.xcactivitylog")
745+
let data = contents.data(using: .utf8)!
746+
try data.gzipped().write(to: url)
747+
return url
748+
}
749+
750+
private func slfContents(from tokens: [Token]) throws -> String {
751+
var classNames: [String] = []
752+
var contents = "SLF"
753+
for token in tokens {
754+
switch token {
755+
case .int(let value):
756+
contents += "\(value)#"
757+
case .className(let value):
758+
classNames.append(value)
759+
contents += "\(value.utf8.count)%\(value)"
760+
case .classNameRef(let value):
761+
let index = try XCTUnwrap(
762+
classNames.firstIndex(of: value),
763+
"Class name \(value) must be emitted before it is referenced"
764+
)
765+
contents += "\(index + 1)@"
766+
case .string(let value):
767+
contents += "\(value.utf8.count)\"\(value)"
768+
case .double(let value):
769+
contents += "\(String(value.bitPattern.byteSwapped, radix: 16))^"
770+
case .null:
771+
contents += "-"
772+
case .list(let value):
773+
contents += "\(value)("
774+
case .json(let value):
775+
contents += "\(value.utf8.count)*\(value)"
776+
}
777+
}
778+
return contents
779+
}
780+
717781
}

‎Tests/XCLogParserTests/LexerTests.swift‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,15 @@ class LexerTests: XCTestCase {
5656
XCTAssertEqual(stringToken, Token.string("Xcode.IDEActivityLogDomainType.BuildLog"))
5757
}
5858

59+
func testTokenizeStringUsesUTF8ByteLength() throws {
60+
let value = "➜ Sources/Bundle+Locali🙂"
61+
let logContents = "SLF0#\(value.utf8.count)\"\(value)1#"
62+
63+
let tokens = try lexer.tokenize(contents: logContents, redacted: false, withoutBuildSpecificInformation: false)
64+
65+
XCTAssertEqual(tokens, [.int(0), .string(value), .int(1)])
66+
}
67+
5968
func testTokenizeDouble() throws {
6069
let logContents = "SLF09#21%IDEActivityLogSection1@39\"Xcode.IDEActivityLogDomainType.BuildLog356098f239dfc041^"
6170
let tokens = try lexer.tokenize(contents: logContents, redacted: false, withoutBuildSpecificInformation: false)

‎Tests/XCLogParserTests/LogLoaderTests.swift‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import Foundation
2+
import Gzip
3+
import XCTest
4+
@testable import XCLogParser
5+
6+
class LogLoaderTests: XCTestCase {
7+
8+
func testLoadFromURLPreservesEmbeddedNullBytes() throws {
9+
let directory = try TestUtils.createRandomTestDir()
10+
defer { try? FileManager.default.removeItem(at: directory) }
11+
let url = directory.appendingPathComponent("test.xcactivitylog")
12+
let value = "Sources/Bundle+Locali\u{0}zation"
13+
let contents = "SLF0#\(value.utf8.count)\"\(value)1#"
14+
let data = contents.data(using: .utf8)!
15+
try data.gzipped().write(to: url)
16+
17+
let loadedContents = try LogLoader().loadFromURL(url)
18+
19+
XCTAssertEqual(loadedContents, contents)
20+
}
21+
}

‎Tests/XCLogParserTests/XCTestManifests.swift‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ extension ActivityParserTests {
77
// to regenerate.
88
static let __allTests__ActivityParserTests = [
99
("testParseActivityLog", testParseActivityLog),
10+
("testParseActivityLogInURLPreservesUTF8ByteLengthStringsAndEmbeddedNullBytes", testParseActivityLogInURLPreservesUTF8ByteLengthStringsAndEmbeddedNullBytes),
1011
("testParseDBGConsoleLog", testParseDBGConsoleLog),
1112
("testParseDVTTextDocumentLocation", testParseDVTTextDocumentLocation),
1213
("testParseIBDocumentMemberLocation", testParseIBDocumentMemberLocation),
@@ -65,10 +66,20 @@ extension LexerTests {
6566
("testTokenizeListNil", testTokenizeListNil),
6667
("testTokenizeString", testTokenizeString),
6768
("testTokenizeStringRedacted", testTokenizeStringRedacted),
69+
("testTokenizeStringUsesUTF8ByteLength", testTokenizeStringUsesUTF8ByteLength),
6870
("testTokenizeStringWithTokenDelimiters", testTokenizeStringWithTokenDelimiters),
6971
]
7072
}
7173

74+
extension LogLoaderTests {
75+
// DO NOT MODIFY: This is autogenerated, use:
76+
// `swift test --generate-linuxmain`
77+
// to regenerate.
78+
static let __allTests__LogLoaderTests = [
79+
("testLoadFromURLPreservesEmbeddedNullBytes", testLoadFromURLPreservesEmbeddedNullBytes),
80+
]
81+
}
82+
7283
extension LogFinderTests {
7384
// DO NOT MODIFY: This is autogenerated, use:
7485
// `swift test --generate-linuxmain`
@@ -138,6 +149,7 @@ public func __allTests() -> [XCTestCaseEntry] {
138149
testCase(IssuesReporterTests.__allTests__IssuesReporterTests),
139150
testCase(LexRedactorTests.__allTests__LexRedactorTests),
140151
testCase(LexerTests.__allTests__LexerTests),
152+
testCase(LogLoaderTests.__allTests__LogLoaderTests),
141153
testCase(LogFinderTests.__allTests__LogFinderTests),
142154
testCase(LogManifestTests.__allTests__LogManifestTests),
143155
testCase(ParserTests.__allTests__ParserTests),

0 commit comments

Comments
 (0)