Advanced file handling in Python

Introduction

In this article you can get acquainted with additional examples of working with files in Python 3.

It is assumed that you have already mastered the material from the previous article Basics of working with files in Python

Create a file files.py and copy the code from the examples into it.

You can run the file with the command

python3 files.py

seek(0): moving to the start of the file

With seek(0) you can put the pointer at the start of the file.

Let's move to the end of the file sites.md

topbicycle.ruheihei.websiteeth1.ru

>>> f = open('sites.md', mode='rt', encoding='utf-8') >>> f.read(3)

top

>>> f.read()

'bicyleheihei.websiteeth1.ru'

>>> f.read()

''

Now with seek() let's put the pointer at the start

>>> f.seek(0)

0

>>> f.read(3)

'top'

Write json to a file

import json # need to get json from somewhere r = urllib.request.urlopen('http://eth1.ru/api/v1/getjson') rr = r.read() rj = json.loads(rr) with open('file.txt', 'w') as f: json.dump(rj, f)

Delete the first few lines of a file

with open('log.txt', 'a') as fin: data = fin.readlines()[1:] with open('new.txt', 'w') as fout: fout.writelines(data)

Writing program output to a file

If you run the script from the terminal, use redirection

python script.py > script.log

In the script itself you can temporarily replace the standard output.

Suppose I make a request to the API

(More about REST API)

import sys … # Save a reference to the original stdout original_stdout = sys.stdout with open("log.txt", "a") as f: sys.stdout = f print(resp.data) sys.stdout = original_stdout

The same applies when what arrives is json and you want to write it out nicely

import sys import json … with open("log.txt", "a") as f: sys.stdout = f print(json.dumps(resp.data, indent=4)) sys.stdout = original_stdout

An example of working with bytes

bmp.py

"""A module for dealing with BMP bitmap image files.""" def write_grayscale(filename, pixels): """Creates and writes a grayscale BMP file. Args: filename: The name of the BMP file to be created. pixels: A rectangular image stored as a sequence of rows. Each row must be an iterable series of integers in the range 0-255. Raises: ValueError: If any of the integer values are out of range. OSError: If the file couldn't be written. """ height = len(pixels) width = len(pixels[0]) with open(filename, 'wb') as bmp: # BMP Header bmp.write(b'BM') size_bookmark = bmp.tell() # The next four bytes hold the filesize as a 32-bit bmp.write(b'\x00\x00\x00\x00') # little-endian integer. Zero placeholder for now. bmp.write(b'\x00\x00') # Unused 16-bit integet - should be zero bmp.write(b'\x00\x00') # Unused 16-bit integet - should be zero pixel_offset_bookmark = bmp.tell() # The next four bytes hold the integer offset to the bmp.write(b'\x00\x00\x00\x00') # pixel data. Zero placeholder for now. # Image Header bmp.write(b'\x28\x00\x00\x00') # Image header size in bytes - 40 decimal bmp.write(_int32_to_bytes(width)) # Image width in pixels bmp.write(_int32_to_bytes(height)) # Image height in pixels bmp.write(b'\x01\x00') # Number of image planes bmp.write(b'\x08\x00') # Bits per pixel 8 for grayscale bmp.write(b'\x00\x00\x00\x00') # No compression bmp.write(b'\x00\x00\x00\x00') # Zero for uncompressed images bmp.write(b'\x00\x00\x00\x00') # Unused pixels per meter bmp.write(b'\x00\x00\x00\x00') # Unused pixels per meter bmp.write(b'\x00\x00\x00\x00') # Use whole color table bmp.write(b'\x00\x00\x00\x00') # All colors are important # Color palette - a linear grayscale for c in range(256): bmp.write(bytes((c, c, c, 0))) # Blue, Green, Red, Zero # Pixel data pixel_data_bookmark = bmp.tell() for row in reversed(pixels): # BMP files are bottom to top row_data = bytes(row) bmp.write(row_data) padding = b'\x00' * ((4 - (len(row) % 4)) % 4) # Pad row to multiple of four bytes bmp.write(padding) # End of file eof_bookmark = bmp.tell() # Fill in file size placeholder bmp.seek(size_bookmark) bmp.write(_int32_to_bytes(eof_bookmark)) # Fill in pixel offset placeholder bmp.seek(pixel_offset_bookmark) bmp.write(_int32_to_bytes(pixel_data_bookmark)) def _int32_to_bytes(i): """Convert an integer to four bytes in little-endian format.""" # &: Bitwise-and # >>: Right-shift return bytes( (i & 0xff, i >> 8 & 0xff, i >> 16 & 0xff, i >> 24 & 0xff) )

fractal.py

import math def mandel(real, imag): """The logarighm of number of iterations needed to determine whether a complex point is in the Mandelbrot set. Args: real: The real coordinate imag: The imaginary coordinate Returns: An integer in the range 1-255. """ x = 0 y = 0 for i in range(1, 257): if x*x + y*y > 4.0: break xt = real + x*x - y*y y = imag + 2.0 * x * y x = xt return int(math.log(i) * 256 / math.log(256)) -1 def mandelbrot(size_x, size_y): """Make an Mandelbrot set image. Args: size_x: Image width size_y: Image height Returns: A list of lists of integers in the range 0-255 """ return [[mandel((3.5 * x / size_x) - 2.5, (2.0 * y / size_y) - 1.0) for x in range(size_x)] for y in range(size_y)]

>>> import fractal >>> pixels = fractal.mandelbrot(448, 256) >>> import reprlib >>> reprlib.repr(pixels)

'[[31, 31, 31, 31, 31, 31, ...], [31, 31, 31, 31, 31, 31, ...], [31, 31, 31, 31, 31, 31, ...], [31, 31, 31, 31, 31, 31, ...], [31, 31, 31, 31, 31, 31, ...], [31, 31, 31, 31, 31, 31, ...], ...]'

>>> import bmp >>> bmp.write_grayscale("mandel.bmp", pixels)

Determining the dimensions of a bmp image

def dimensions(filename): """Determine the dimensions in pixels of a BMP image. Args: filename: The filename of a BMP file. Returns: A tuple containing two integers with the width and height in pixels. Raises: ValueError: If the file was not a BMP file. OSError: If there was a problem reading the file. """ with open(filename, 'rb') as f: magic = f.read(2) if magic != b'BM': raise ValueError(f"{filename} is not a BMP file") f.seek(18) width_bytes = f.read(4) height_bytes = f.read(4) return ( _bytes_to_int32(width_bytes), _bytes_to_int32(height_bytes)) def _bytes_to_int32(b): "Convert a bytes object containing four bytes into an integer." return b[0] | (b[1] << 8) | (b[2] << 16) | (b[3] << 24)

>>> import bmp >>> bmp.dimensions("mandel.bmp")

(448, 256)

Detect file encodings

An example script for detecting file encodings. To learn how to create files in different encodings in Linux read here

python -m pip install python-magic

import magic def get_encoding(sample): blob = open(sample, 'rb').read() m = magic.open(magic.MAGIC_MIME_ENCODING) m.load() encoding = m.buffer(blob) return encoding files = ['utf-8-file', 'windows-1251-file', 'shift-jis-file'] for f in files: print(get_encoding(f))

utf-8 iso-8859-1 unknown-8bit

There are still problems with detecting SHIFT-JIS

The path to a file

More about the library pathlib you can read here

python -m pip install pathlib

import pathlib from pathlib import Path dir_path = pathlib.Path.cwd() print(dir_path)

/home/andrei/sandbox/python/file_path

Read a file from another directory

Suppose we are in the directory one of the project file_path:

file_path/ ├── one │ └── path.py └── two └── sites.txt

cat ../two/sites.txt

www.heihei.website

To read the file sites.txt with Python, the library will help pathlib

import pathlib from pathlib import Path dir_path = pathlib.Path.cwd() path = Path(dir_path, "..", "two", "sites.txt") with open(path, "r") as f: sites = f.read() print(sites)

python path.py

www.heihei.website

Find a string

Let's look at the file sites.md and try to find the string heihei.website

topbicycle.ru heihei.website eht1.ru

We'll use the fact that the context manager returns an iterator and go through the file with the method next()

with open("sites.md", "r") as f: while True: try: line = str(next(f)) print(line) except StopIteration: raise ValueError("End of file") if line == "heihei.website\n": print("Found heihei.website!")

topbicycle.ru heihei.website Found heihei.website! eth1.ru Traceback (most recent call last): File "/mnt/c/Users/Andrei/readline_ex.py", line 4, in <module> line = str(next(f)) ^^^^^^^ StopIteration During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/mnt/c/Users/Andrei/readline_ex.py", line 7, in <module> raise ValueError("End of file") ValueError: End of file

With a for loop

with open("sites.md", "r") as f: for line in f: if line == "heihei.website\n": print("Found heihei.website") else: print("Some other line found")

Some other line found Found heihei.website Some other line found

Delete a specific line

Let's look at the file sites.md and try to delete all the lines heihei.website

topbicycle.ru heihei.website eht1.ru

with open("sites.md", "r") as f: with open("new_file.txt", "w") as nf: for line in f: if line != "heihei.website\n": nf.write(line) with open("new_file.txt", "r") as nf: print(nf.readlines())

['topbicycle.ru\n', 'eth1.ru\n']

Banner image

Delete everything between two lines

Let's look at the file sites.md

topbicycle.ru heihei.website <bikes> Forward Stark Stels </bikes> eth1.ru devhops.ru

Let's delete everything between the <bikes> and </bikes> tags and write it to a new file.

Including the tags themselves:

copy = True with open("sites.md", "r") as f: print(f.readlines()) with open("sites.md", "r") as f: with open("clean.md", "w") as nf: for line in f: if line == "<bikes>\n": copy = False if copy: nf.write(line) if line == "</bikes>\n": copy = True with open("clean.md", "r") as nf: print(nf.readlines())

['topbicycle.ru\n', 'heihei.website\n', '<bikes>\n', 'Forward\n', 'Stark\n', 'Stels\n', '</bikes>\n', 'eth1.ru\n', 'devhops.ru\n'] ['topbicycle.ru\n', 'heihei.website\n', 'eth1.ru\n', 'devhops.ru\n']

Keeping the tags:

copy = True with open("sites.md", "r") as f: print(f.readlines()) with open("sites.md", "r") as f: with open("clean.md", "w") as nf: for line in f: if line == "</bikes>\n": copy = True if copy: nf.write(line) if line == "<bikes>\n": copy = False with open("clean.md", "r") as nf: print(nf.readlines())

['topbicycle.ru\n', 'heihei.website\n', '<bikes>\n', 'Forward\n', 'Stark\n', 'Stels\n', '</bikes>\n', 'eth1.ru\n', 'devhops.ru\n'] ['topbicycle.ru\n', 'heihei.website\n', '<bikes>\n', '</bikes>\n', 'eth1.ru\n', 'devhops.ru\n']

A more universal version using functions

Here the changes will be written straight to the source file.

def main(): path = "sites.md" tag = "<bikes>" lines = readlines(path) delete_between_tags(path, lines, tag) def readlines(path): with open(path, "r") as f: lines = f.readlines() return lines def delete_between_tags(path, lines, tag): # or strip line.strip("\n") in a loop tag = str(tag) + "\n" taglist = list(tag) taglist.insert(1, "/") endtag = "".join(taglist) with open(path, "w") as f: copy = True for line in lines: if line == tag: copy = False if copy: f.write(line) if line == endtag: copy = True if __name__ == "__main__": main()

A tag does not have to be at the beginning of a line, so the script can be improved.

For example, in such a file the previous script will not delete anything.

heihei.website <topbicycle.ru> <bikes> Forward Stark Stels </bikes> </topbicycle.ru> eth1.ru devhops.ru

If we are sure that each tag is written on a separate line, it is enough to replace line = tag with tag in line and the same for endtag.

for line in lines: if tag in line: copy = False if copy: f.write(line) if endtag in line: copy = True

The line

tag = str(tag) + "\n"

also becomes unnecessary.

heihei.website <topbicycle.ru> </topbicycle.ru> eth1.ru devhops.ru

Append after a tag

Let's look at the file sites.md

heihei.website <topbicycle.ru> <bikes> Forward Stark Stels </bikes> </topbicycle.ru> eth1.ru devhops.ru

Let's append the following text after the <bikes> tag

<german> Cube Drossiger Ghost </german>

def main(): path = "sites.md" tag = "<bikes>" text = """ <german> Cube Drossiger Ghost </german> """ lines = readlines(path) append_after_tag(path, lines, tag, text) def readlines(path): with open(path, "r") as f: lines = f.readlines() return lines def append_after_tag(path, lines, tag, text): tag = str(tag) + "\n" taglist = list(tag) taglist.insert(1, "/") endtag = "".join(taglist) with open(path, "w") as f: write_flag = True for line in lines: f.write(line) if tag in line: if write_flag == True: f.write(text) write_flag = False if __name__ == "__main__": main()

heihei.website <topbicycle.ru> <bikes> <german> Cube Drossiger Ghost </german> Forward Stark Stels </bikes> </topbicycle.ru> eth1.ru devhops.ru

Remove the file extension

Let's look at the script remove_ex.py and the file openapi.yaml in the same directory. First let's look at using os.path.splitext()

demo `-- example |-- openapi.yaml `-- remove_ex.py

import os path = "openapi.yaml" print(os.path.splitext(path)[0])

We'll run the script from the current directory and from the parent one

python remove_ex.py

python example/remove_ex.py

openapi

Now suppose the file needs to be copied at the end of the script with shutil

import os import shutil path = "openapi.yaml" print(os.path.splitext(path)[0]) shutil.copyfile(path, "backup.yaml")

python remove_ex.py

openapi

A file will appear in the current directory backup.yaml

demo/ `-- example |-- backup.yaml |-- openapi.yaml `-- remove_ex.py

python example/remove_ex.py

FileNotFoundError: [Errno 2] No such file or directory: 'openapi.yaml'

When run from the current directory everything was fine, but from the parent directory shutil no longer finds the source file.

Let's use the full path to openapi.yaml

import os import shutil path = "openapi.yaml" path = os.path.join(os.path.dirname(__file__), path) print(os.path.splitext(path)[0]) shutil.copyfile(path, "backup.yaml")

From the current directory everything is still fine and backup.yaml is created in it as well.

python remove_ex.py

openapi

Nevertheless, already in this case when running from PyCharm the result will be

C:\Users\Andrei\demo\example\openapi openapi

When running from the parent directory

python example/remove_ex.py

example\openapi

backup.yaml is created in the parent directory

demo/ |-- backup.yaml `-- example |-- openapi.yaml `-- remove_ex.py

To extract only the file name, the following will help os.path.basename()

import os import shutil path = "openapi.yaml" path = os.path.join(os.path.dirname(__file__), path) print(os.path.splitext(os.path.basename(path))[0]) shutil.copyfile(path, "backup.yaml")

pathlib.Path().stem

Now let's look at using pathlib.Path().stem

demo `-- example |-- openapi.yaml `-- remove_ex.py

from pathlib import Path path = "openapi.yaml" print(Path(path).stem)

We'll run from the current directory and from the parent one

python remove_ex.py

python example/remove_ex.py

openapi

Now suppose the file needs to be copied at the end of the script with shutil

import shutil from pathlib import Path path = "openapi.yaml" print(Path(path).stem) shutil.copyfile(path, "backup.yaml")

python remove_ex.py

openapi

A file will appear in the current directory backup.yaml

demo/ `-- example |-- backup.yaml |-- openapi.yaml `-- remove_ex.py

python example/remove_ex.py

FileNotFoundError: [Errno 2] No such file or directory: 'openapi.yaml'

When run from the current directory everything was fine, but from the parent directory shutil no longer finds the source file.

Let's use the full path to openapi.yaml

import shutil from pathlib import Path path = "openapi.yaml" path = os.path.join(os.path.dirname(__file__), path) print(Path(path).stem) shutil.copyfile(path, "backup.yaml")

From the current directory everything is still fine and backup.yaml is created in it as well.

python remove_ex.py

openapi

When running from the parent directory

python example/remove_ex.py

openapi

backup.yaml is created in the parent directory

demo/ |-- backup.yaml `-- example |-- openapi.yaml `-- remove_ex.py

An example of a script that:

Converts a yaml file to json

Saves the original versions to the backup directory

Indicates the date and time of saving in the name of the backup files.

import yaml import json import time import shutil import os YAML_FILE_PATH = os.path.join(os.path.dirname(__file__), "openapi.yaml") JSON_FILE_PATH = os.path.join(os.path.dirname(__file__), "swagger.json") BACKUP_PATH = os.path.join(os.path.dirname(__file__), "backup") def yaml_to_json(yaml_file_path, json_file_path): with open(yaml_file_path, 'r') as yaml_file: yaml_data = yaml.safe_load(yaml_file) with open(json_file_path, 'w') as json_file: json.dump(yaml_data, json_file, indent=2) def backup(*args): for path in args: timestr = time.strftime("-%Y%m%d-%H%M%S") name = os.path.splitext(os.path.basename(path))[0] ext = os.path.splitext(os.path.basename(path))[1] name_with_time = name + timestr + ext dest = os.path.join(BACKUP_PATH, name_with_time) shutil.copyfile(path, dest) if __name__ == "__main__": if not os.path.exists(BACKUP_PATH): os.makedirs(BACKUP_PATH) backup(YAML_FILE_PATH, JSON_FILE_PATH) yaml_to_json(YAML_FILE_PATH, JSON_FILE_PATH)

Article author: Andrei Olegovich