Your suggestion has been really helpful! As you suggested, indexing a simple frequency array instead of a map results in better performance. Thanx!

Here is the working version for Swift:

var t = Int(readLine()!)!

for _ in (0..<t) {
    var strArr = Array(readLine()!.characters)

    if strArr.count % 2 != 0 {
        print("-1")
        continue
    }      

    var letter = [Int](count: 26, repeatedValue: 0)
    var count = 0

    for i in (0..<strArr.count / 2) {
        var charNum = Int(Array(String(strArr[i]).utf8)[0])
        letter[charNum - 97] = letter[charNum - 97] + 1
    }

    for i in (strArr.count / 2..<strArr.count) {
        var charNum = Int(Array(String(strArr[i]).utf8)[0])
        letter[charNum - 97] = letter[charNum - 97] - 1
    }

    for i in (0..<26) {
        count = count + abs(letter[i])
    }

    print(String(count / 2))

}