package com.jmcejuela.ml
import java.io.InputStream
import java.io.Writer
import scala.io.Codec
import scala.io.Source
import Table._
/**
* Class to represent tables with a fixed size of columns. All rows have the same columns.
*/
class Table(val rows: Seq[Row]) {
lazy val numDiffColumns = rows.foldLeft(Set[Int]())((set, row) => set + row.size)
def toTSV(out: Writer) {
if (rows.isEmpty) out.write(TableEmpty.toString)
else {
out.write(writeLineTSV(rows.head.map(_.name))) //header
rows.foreach(r => out.write(writeLineTSV(r.map(_.value))))
out.close
}
}
/**
* Get a Table with only the given columns.
*/
def filterColumnsByName(columnNames: Set[String]): Table = {
val existingNames = rows.head.map(_.name).toSet
assert(columnNames.forall(n => existingNames.contains(n)), "You want to include column names that do not exist")
new Table(rows.map { row => row.filter(col => columnNames.contains(col.name)) })
}
}
object TableEmpty extends Table(Seq.empty) {
override def toString = "Table(Empty)"
}
object Table {
def apply(rows: Row*) = new Table(rows)
type Row = Array[Column]
/**
* Column representation. Note that each column has a name and a value. Since the class Table
* is a sequence of rows which are a size-fixed array of columns, the name field is redundant
* for Table. However, this column representation could be used in the future to support
* schemata-less tables.
*/
case class Column(name: String, value: String)
private def parseLineTSV(line: String) = line.split("\t")
private def writeLineTSV(line: Seq[String]) = line.mkString("", "\t", "\n")
/**
* It is assumed that the first row gives the names to the columns
*/
def fromTSV(in: InputStream)(implicit encoding: Codec = Codec.UTF8): Table = {
val linesIt = Source.fromInputStream(in).getLines
if (linesIt.isEmpty) TableEmpty
else {
val columnNames = parseLineTSV(linesIt.next)
val padding = {
//add padding of empty columns-fields to lines that do not include last fields because they are empty
def infinite[A](x: A): Stream[A] = x #:: infinite(x)
infinite("")
}
val rows = linesIt.map { line =>
((0 until columnNames.size).zip(parseLineTSV(line) ++: padding).map { case (index, field) => Column(columnNames(index), field) }).toArray
}.toStream
new Table(rows)
}
}
}
4 回答
我认为您最好使用excel中的导入工具,但您可能必须手动将文件扩展名更改为 csv .
导入时,请务必为具有这些值的所有列选择文本 .
我的问题实际上至少是一个副本:
1)Stop Excel from automatically converting certain text values to dates
2)Excel: Default to TEXT rather than GENERAL when opening a .csv file
Excel的可能解决方案是:1)使用特殊的双引号编写字段,如
"May 16, 2011"
作为"=""May 16, 2011"""
或2)使用外部数据向导导入csv / tsv文件,然后手动选择要作为TEXT而不是GENERAL读取的列(可以将字段转换为日期)至于我的用例,我只使用Excel删除了一些列 . 没有一个解决方案对我有吸引力,因为我不想用特殊引号重写tsv文件,因为我有数百列,我不想手动选择每个作为TEXT读取 .
因此,我写了一个scala脚本来按列名过滤tsv文件:
在excel中写01-04而不是1-4
我在excel中有一个“文本”格式的单元格,其中填充了一个化学值,其值为“8013-07-8”,正在重新格式化为日期格式 . 为了解决这个问题,我将单个引号连接到值的开头,并在查看结果时正确呈现 . 当您单击单元格时,您会看到带前缀的单引号,但至少我已将其视为日期 .