构建一个sklearn文本分类器并使用coremltools进行转换

1 回答

• 1

  现在，如果要将其转换为.mlmodel格式，则无法在管道中包含tf-idf矢量化程序 . 解决这个问题的方法是分别对数据进行矢量化，然后使用矢量化数据训练模型（线性SVC，随机森林......） . 然后，您需要计算设备上的tf-idf表示，然后将其插入模型中 . 这是我写的tf-idf函数的副本 .

  func tfidf(document: String) -> MLMultiArray{
      let wordsFile = Bundle.main.path(forResource: "words_ordered", ofType: "txt")
      let dataFile = Bundle.main.path(forResource: "data", ofType: "txt")
      do {
          let wordsFileText = try String(contentsOfFile: wordsFile!, encoding: String.Encoding.utf8)
          var wordsData = wordsFileText.components(separatedBy: .newlines)
          let dataFileText = try String(contentsOfFile: dataFile!, encoding: String.Encoding.utf8)
          var data = dataFileText.components(separatedBy: .newlines)
          let wordsInMessage = document.split(separator: " ")
          var vectorized = try MLMultiArray(shape: [NSNumber(integerLiteral: wordsData.count)], dataType: MLMultiArrayDataType.double)
          for i in 0..<wordsData.count{
              let word = wordsData[i]
              if document.contains(word){
                  var wordCount = 0
                  for substr in wordsInMessage{
                      if substr.elementsEqual(word){
                          wordCount += 1
                      }
                  }
                  let tf = Double(wordCount) / Double(wordsInMessage.count)
                  var docCount = 0
                  for line in data{
                      if line.contains(word) {
                          docCount += 1
                      }
                  }
                  let idf = log(Double(data.count) / Double(docCount))
                  vectorized[i] = NSNumber(value: tf * idf)
              } else {
                  vectorized[i] = 0.0
              }
          }
          return vectorized
      } catch {
          return MLMultiArray()
      }
  }
  

  编辑：在http://gokulswamy.me/imessage-spam-detection/撰写了关于如何做到这一点的完整帖子 .

  回复于 2024-04-26T15:48:20+08:00