使用python以相反的顺序读取文件

如何使用python以相反的顺序读取文件?我想从最后一行到第一行读取一个文件 .

回答(18)

2 years ago

for line in reversed(open("filename").readlines()):
    print line.rstrip()

在Python 3中:

for line in reversed(list(open("filename"))):
    print(line.rstrip())

2 years ago

作为生成器编写的正确,有效的答案 .

import os

def reverse_readline(filename, buf_size=8192):
    """a generator that returns the lines of a file in reverse order"""
    with open(filename) as fh:
        segment = None
        offset = 0
        fh.seek(0, os.SEEK_END)
        file_size = remaining_size = fh.tell()
        while remaining_size > 0:
            offset = min(file_size, offset + buf_size)
            fh.seek(file_size - offset)
            buffer = fh.read(min(remaining_size, buf_size))
            remaining_size -= buf_size
            lines = buffer.split('\n')
            # the first line of the buffer is probably not a complete line so
            # we'll save it and append it to the last line of the next buffer
            # we read
            if segment is not None:
                # if the previous chunk starts right from the beginning of line
                # do not concact the segment to the last line of new chunk
                # instead, yield the segment first 
                if buffer[-1] != '\n':
                    lines[-1] += segment
                else:
                    yield segment
            segment = lines[0]
            for index in range(len(lines) - 1, 0, -1):
                if lines[index]:
                    yield lines[index]
        # Don't yield None if the file was empty
        if segment is not None:
            yield segment

2 years ago

这样的事情怎么样:

import os


def readlines_reverse(filename):
    with open(filename) as qfile:
        qfile.seek(0, os.SEEK_END)
        position = qfile.tell()
        line = ''
        while position >= 0:
            qfile.seek(position)
            next_char = qfile.read(1)
            if next_char == "\n":
                yield line[::-1]
                line = ''
            else:
                line += next_char
            position -= 1
        yield line[::-1]


if __name__ == '__main__':
    for qline in readlines_reverse(raw_input()):
        print qline

由于文件是按相反的顺序逐字读取的,因此只要单个行适合内存,它甚至可以在非常大的文件上工作 .

2 years ago

你也可以使用python模块 file_read_backwards .

安装后,通过 pip install file_read_backwards (v1.2.1),您可以通过以下内容高效的方式向后(按行)读取整个文件:

#!/usr/bin/env python2.7

from file_read_backwards import FileReadBackwards

with FileReadBackwards("/path/to/file", encoding="utf-8") as frb:
    for l in frb:
         print l

它支持“utf-8”,“latin-1”和“ascii”编码 .

python3也支持 . 更多文档可在http://file-read-backwards.readthedocs.io/en/latest/readme.html找到

2 years ago

for line in reversed(open("file").readlines()):
    print line.rstrip()

如果您使用的是Linux,则可以使用 tac 命令 .

$ tac file

你可以在ActiveState herehere找到2个食谱

2 years ago

import re

def filerev(somefile, buffer=0x20000):
  somefile.seek(0, os.SEEK_END)
  size = somefile.tell()
  lines = ['']
  rem = size % buffer
  pos = max(0, (size // buffer - 1) * buffer)
  while pos >= 0:
    somefile.seek(pos, os.SEEK_SET)
    data = somefile.read(rem + buffer) + lines[0]
    rem = 0
    lines = re.findall('[^\n]*\n?', data)
    ix = len(lines) - 2
    while ix > 0:
      yield lines[ix]
      ix -= 1
    pos -= buffer
  else:
    yield lines[0]

with open(sys.argv[1], 'r') as f:
  for line in filerev(f):
    sys.stdout.write(line)

2 years ago

在这里你可以找到我的实现,你可以通过更改“缓冲区”变量来限制ram的使用,有一个错误,程序在开头打印一个空行 .

如果没有超过缓冲区字节的新行,ram的使用可能会增加,“leak”变量将增加,直到看到一个新行(“\ n”) .

这也适用于16 GB文件,这比我的总内存大 .

import os,sys
buffer = 1024*1024 # 1MB
f = open(sys.argv[1])
f.seek(0, os.SEEK_END)
filesize = f.tell()

division, remainder = divmod(filesize, buffer)
line_leak=''

for chunk_counter in range(1,division + 2):
    if division - chunk_counter < 0:
        f.seek(0, os.SEEK_SET)
        chunk = f.read(remainder)
    elif division - chunk_counter >= 0:
        f.seek(-(buffer*chunk_counter), os.SEEK_END)
        chunk = f.read(buffer)

    chunk_lines_reversed = list(reversed(chunk.split('\n')))
    if line_leak: # add line_leak from previous chunk to beginning
        chunk_lines_reversed[0] += line_leak

    # after reversed, save the leakedline for next chunk iteration
    line_leak = chunk_lines_reversed.pop()

    if chunk_lines_reversed:
        print "\n".join(chunk_lines_reversed)
    # print the last leaked line
    if division - chunk_counter < 0:
        print line_leak

2 years ago

谢谢@srohde的答案 . 它有一个小错误检查带有'is'运算符的换行符,我无法用1声誉评论答案 . 此外,我想管理外面的文件,因为这使我能够将我的rambling嵌入到luigi任务中 .

我需要改变的形式是:

with open(filename) as fp:
    for line in fp:
        #print line,  # contains new line
        print '>{}<'.format(line)

我很乐意改为:

with open(filename) as fp:
    for line in reversed_fp_iter(fp, 4):
        #print line,  # contains new line
        print '>{}<'.format(line)

这是一个修改后的答案,需要文件句柄并保留换行符:

def reversed_fp_iter(fp, buf_size=8192):
    """a generator that returns the lines of a file in reverse order
    ref: https://stackoverflow.com/a/23646049/8776239
    """
    segment = None  # holds possible incomplete segment at the beginning of the buffer
    offset = 0
    fp.seek(0, os.SEEK_END)
    file_size = remaining_size = fp.tell()
    while remaining_size > 0:
        offset = min(file_size, offset + buf_size)
        fp.seek(file_size - offset)
        buffer = fp.read(min(remaining_size, buf_size))
        remaining_size -= buf_size
        lines = buffer.splitlines(True)
        # the first line of the buffer is probably not a complete line so
        # we'll save it and append it to the last line of the next buffer
        # we read
        if segment is not None:
            # if the previous chunk starts right from the beginning of line
            # do not concat the segment to the last line of new chunk
            # instead, yield the segment first
            if buffer[-1] == '\n':
                #print 'buffer ends with newline'
                yield segment
            else:
                lines[-1] += segment
                #print 'enlarged last line to >{}<, len {}'.format(lines[-1], len(lines))
        segment = lines[0]
        for index in range(len(lines) - 1, 0, -1):
            if len(lines[index]):
                yield lines[index]
    # Don't yield None if the file was empty
    if segment is not None:
        yield segment

2 years ago

一个简单的函数来创建第二个文件reverse(仅限linux):

import os
def tac(file1, file2):
     print(os.system('tac %s > %s' % (file1,file2)))

如何使用

tac('ordered.csv', 'reversed.csv')
f = open('reversed.csv')

2 years ago

如果您担心文件大小/内存使用情况,内存映射文件和向后扫描换行是一种解决方案:

How to search for a string in text files?

2 years ago

对于具有不适合内存的大文件的情况,接受的答案将不起作用(这种情况并不罕见) .

正如其他人所指出的那样,@srohde answer看起来不错,但它有下一个问题:

  • openning文件看起来是多余的,当我们可以传递文件对象并将其留给用户来决定应该读取哪个编码时,

  • 即使我们重构接受文件对象,它也不适用于所有编码:我们可以选择 utf-8 编码的文件和非ascii内容

й

传递 buf_size 等于 1 并将拥有

UnicodeDecodeError: 'utf8' codec can't decode byte 0xb9 in position 0: invalid start byte

当然文本可能会更大但是 buf_size 可能会被拾取,因此它会导致像上面那样的混淆错误,

  • 我们无法指定自定义行分隔符,

  • 我们不能选择保持行分隔符 .

所以考虑到所有这些问题,我写了不同的功能:

  • 一个用于字节流的,

  • 第二个使用文本流并将其基础字节流委托给第一个并对结果行进行解码 .

首先让我们定义下一个效用函数:

ceil_division 用于使用天花板进行划分(与带有楼层的标准 // 划分相比,更多信息可以在this thread中找到)

def ceil_division(left_number, right_number):
    """
    Divides given numbers with ceiling.
    """
    return -(-left_number // right_number)

split 用于按给定分隔符拆分字符串并具有保留它的能力:

def split(string, separator, keep_separator):
    """
    Splits given string by given separator.
    """
    parts = string.split(separator)
    if keep_separator:
        *parts, last_part = parts
        parts = [part + separator for part in parts]
        if last_part:
            return parts + [last_part]
    return parts

read_batch_from_end 从二进制流的右端读取批处理

def read_batch_from_end(byte_stream, size, end_position):
    """
    Reads batch from the end of given byte stream.
    """
    if end_position > size:
        offset = end_position - size
    else:
        offset = 0
        size = end_position
    byte_stream.seek(offset)
    return byte_stream.read(size)

之后我们可以定义以相反顺序读取字节流的函数

import functools
import itertools
import os
from operator import methodcaller, sub


def reverse_binary_stream(byte_stream, batch_size=None,
                          lines_separator=None,
                          keep_lines_separator=True):
    if lines_separator is None:
        lines_separator = (b'\r', b'\n', b'\r\n')
        lines_splitter = methodcaller(str.splitlines.__name__,
                                      keep_lines_separator)
    else:
        lines_splitter = functools.partial(split,
                                           separator=lines_separator,
                                           keep_separator=keep_lines_separator)
    stream_size = byte_stream.seek(0, os.SEEK_END)
    if batch_size is None:
        batch_size = stream_size or 1
    batches_count = ceil_division(stream_size, batch_size)
    remaining_bytes_indicator = itertools.islice(
            itertools.accumulate(itertools.chain([stream_size],
                                                 itertools.repeat(batch_size)),
                                 sub),
            batches_count)
    try:
        remaining_bytes_count = next(remaining_bytes_indicator)
    except StopIteration:
        return
    batch = read_batch_from_end(byte_stream,
                                size=batch_size,
                                end_position=remaining_bytes_count)
    segment, *lines = lines_splitter(batch)
    # for Python 3.2 or earlier versions next statement can be replaced with
    # for line in reversed(lines): yield line
    yield from reversed(lines)
    for remaining_bytes_count in remaining_bytes_indicator:
        batch = read_batch_from_end(byte_stream,
                                    size=batch_size,
                                    end_position=remaining_bytes_count)
        lines = lines_splitter(batch)
        if batch.endswith(lines_separator):
            yield segment
        else:
            lines[-1] += segment
        segment, *lines = lines
        # for Python 3.2 or earlier versions next statement can be replaced with
        # for line in reversed(lines): yield line
        yield from reversed(lines)
    yield segment

最后一个用于反转文本文件的函数可以定义如下:

import codecs


def reverse_file(file, batch_size=None, 
                 lines_separator=None,
                 keep_lines_separator=True):
    encoding = file.encoding
    if lines_separator is not None:
        lines_separator = lines_separator.encode(encoding)
    yield from map(functools.partial(codecs.decode,
                                     encoding=encoding),
                   reverse_binary_stream(
                           file.buffer,
                           batch_size=batch_size,
                           lines_separator=lines_separator,
                           keep_lines_separator=keep_lines_separator))

测试

准备工作

我使用fsutil command生成了4个文件:

  • empty.txt没有内容,大小为0MB

  • tiny.txt,大小为1MB

  • small.txt,大小为10MB

  • large.txt,大小为50MB

我也重构了@srohde解决方案来处理文件对象而不是文件路径 .

测试脚本

from timeit import Timer

repeats_count = 7
number = 1
create_setup = ('from collections import deque\n'
                'from __main__ import reverse_file, reverse_readline\n'
                'file = open("{}")').format
srohde_solution = ('with file:\n'
                   '    deque(reverse_readline(file,\n'
                   '                           buf_size=8192),'
                   '          maxlen=0)')
azat_ibrakov_solution = ('with file:\n'
                         '    deque(reverse_file(file,\n'
                         '                       lines_separator="\\n",\n'
                         '                       keep_lines_separator=False,\n'
                         '                       batch_size=8192), maxlen=0)')
print('reversing empty file by "srohde"',
      min(Timer(srohde_solution,
                create_setup('empty.txt')).repeat(repeats_count, number)))
print('reversing empty file by "Azat Ibrakov"',
      min(Timer(azat_ibrakov_solution,
                create_setup('empty.txt')).repeat(repeats_count, number)))
print('reversing tiny file (1MB) by "srohde"',
      min(Timer(srohde_solution,
                create_setup('tiny.txt')).repeat(repeats_count, number)))
print('reversing tiny file (1MB) by "Azat Ibrakov"',
      min(Timer(azat_ibrakov_solution,
                create_setup('tiny.txt')).repeat(repeats_count, number)))
print('reversing small file (10MB) by "srohde"',
      min(Timer(srohde_solution,
                create_setup('small.txt')).repeat(repeats_count, number)))
print('reversing small file (10MB) by "Azat Ibrakov"',
      min(Timer(azat_ibrakov_solution,
                create_setup('small.txt')).repeat(repeats_count, number)))
print('reversing large file (50MB) by "srohde"',
      min(Timer(srohde_solution,
                create_setup('large.txt')).repeat(repeats_count, number)))
print('reversing large file (50MB) by "Azat Ibrakov"',
      min(Timer(azat_ibrakov_solution,
                create_setup('large.txt')).repeat(repeats_count, number)))

注意:我使用 collections.deque 类来排气发电机 .

输出

对于Windows 10上的PyPy 3.5:

reversing empty file by "srohde" 8.31e-05
reversing empty file by "Azat Ibrakov" 0.00016090000000000028
reversing tiny file (1MB) by "srohde" 0.160081
reversing tiny file (1MB) by "Azat Ibrakov" 0.09594989999999998
reversing small file (10MB) by "srohde" 8.8891863
reversing small file (10MB) by "Azat Ibrakov" 5.323388100000001
reversing large file (50MB) by "srohde" 186.5338368
reversing large file (50MB) by "Azat Ibrakov" 99.07450229999998

对于Windows 10上的CPython 3.5:

reversing empty file by "srohde" 3.600000000000001e-05
reversing empty file by "Azat Ibrakov" 4.519999999999958e-05
reversing tiny file (1MB) by "srohde" 0.01965560000000001
reversing tiny file (1MB) by "Azat Ibrakov" 0.019207699999999994
reversing small file (10MB) by "srohde" 3.1341862999999996
reversing small file (10MB) by "Azat Ibrakov" 3.0872588000000007
reversing large file (50MB) by "srohde" 82.01206720000002
reversing large file (50MB) by "Azat Ibrakov" 82.16775059999998

因此,我们可以看到它表现得像原始解决方案,但更为通用,没有上面列出的缺点 .


广告

我已将此添加到lz package lz package(需要Python 3.5),其中包含许多经过良好测试的功能/迭代实用程序 .

可以用喜欢

import io
 from lz.iterating import reverse
 ...
 with open('path/to/file') as file:
     for line in reverse(file, batch_size=io.DEFAULT_BUFFER_SIZE):
         print(line)

它支持all standard encodings(可能除了 utf-7 ,因为我很难定义a strategy来生成可用它编码的字符串) .

2 years ago

def reverse_lines(filename):
    y=open(filename).readlines()
    return y[::-1]

2 years ago

在处理文件时始终使用 with ,因为它会为您处理所有事情:

with open('filename', 'r') as f:
    for line in reversed(f.readlines()):
        print line

或者在Python 3中:

with open('filename', 'r') as f:
    for line in reversed(list(f.readlines())):
        print(line)

2 years ago

您需要先以读取格式打开文件,将其保存到变量中,然后以写入格式打开第二个文件,您可以使用[:: - 1]切片编写或附加变量,从而完全反转文件 . 您还可以使用readlines()将其转换为可以操作的行列表

def copy_and_reverse(filename, newfile):
    with open(filename) as file:
        text = file.read()
    with open(newfile, "w") as file2:
        file2.write(text[::-1])

2 years ago

在做任何事情之前,大多数答案都需要读取整个文件 . 该样本从最后读取越来越大的样本 .

我只看到MuratYükselen's answer while writing this answer. It'几乎一样,我认为这是件好事 . 下面的示例还处理\ r并在每一步增加其缓冲区大小 . 我也有一些unit tests支持这个代码 .

def readlines_reversed(f):
    """ Iterate over the lines in a file in reverse. The file must be
    open in 'rb' mode. Yields the lines unencoded (as bytes), including the
    newline character. Produces the same result as readlines, but reversed.
    If this is used to reverse the line in a file twice, the result is
    exactly the same.
    """
    head = b""
    f.seek(0, 2)
    t = f.tell()
    buffersize, maxbuffersize = 64, 4096
    while True:
        if t <= 0:
            break
        # Read next block
        buffersize = min(buffersize * 2, maxbuffersize)
        tprev = t
        t = max(0, t - buffersize)
        f.seek(t)
        lines = f.read(tprev - t).splitlines(True)
        # Align to line breaks
        if not lines[-1].endswith((b"\n", b"\r")):
            lines[-1] += head  # current tail is previous head
        elif head == b"\n" and lines[-1].endswith(b"\r"):
            lines[-1] += head  # Keep \r\n together
        elif head:
            lines.append(head)
        head = lines.pop(0)  # can be '\n' (ok)
        # Iterate over current block in reverse
        for line in reversed(lines):
            yield line
    if head:
        yield head

2 years ago

逐行读取文件,然后以相反的顺序将其添加到列表中 .

这是一个代码示例:

reverse = []
with open("file.txt", "r") as file:
    for line in file:
        line = line.strip()
         reverse[0:0] = line

2 years ago

import sys
f = open(sys.argv[1] , 'r')
for line in f.readlines()[::-1]:
    print line

2 years ago

我不得不在前一段时间做这个并使用下面的代码 . 它管到壳 . 恐怕我不再拥有完整的剧本了 . 如果您使用的是unixish操作系统,则可以使用“tac”,但是例如Mac OSX tac命令不起作用,使用tail -r . 下面的代码片段测试您所在的平台,并相应地调整命令

# We need a command to reverse the line order of the file. On Linux this
# is 'tac', on OSX it is 'tail -r'
# 'tac' is not supported on osx, 'tail -r' is not supported on linux.

if sys.platform == "darwin":
    command += "|tail -r"
elif sys.platform == "linux2":
    command += "|tac"
else:
    raise EnvironmentError('Platform %s not supported' % sys.platform)