Advanced file handling in Python
Introduction
In this article you can get acquainted with additional examples of working with files in Python 3.
It is assumed that you have already mastered the material from the previous article
Basics of working with files in Python
Create a file
files.py
and copy the code from the examples into it.
You can run the file with the command
python3 files.py
seek(0): moving to the start of the file
With seek(0) you can put the pointer at the start of the file.
Let's move to the end of the file
sites.md
topbicycle.ruheihei.websiteeth1.ru
>>> f = open('sites.md', mode='rt', encoding='utf-8') >>> f.read(3)
top
>>> f.read()
'bicyleheihei.websiteeth1.ru'
>>> f.read()
''
Now with seek() let's put the pointer at the start
>>> f.seek(0)
0
>>> f.read(3)
'top'
Write json to a file
import json
# need to get json from somewhere
r = urllib.request.urlopen('http://eth1.ru/api/v1/getjson')
rr = r.read()
rj = json.loads(rr)
with open('file.txt', 'w') as f:
json.dump(rj, f)
Delete the first few lines of a file
with open('log.txt', 'a') as fin:
data = fin.readlines()[1:]
with open('new.txt', 'w') as fout:
fout.writelines(data)
Writing program output to a file
If you run the script from the terminal, use redirection
python script.py > script.log
In the script itself you can temporarily replace the standard output.
Suppose I make a request to the API
(More about REST API)
import sys … # Save a reference to the original stdout original_stdout = sys.stdout with open("log.txt", "a") as f: sys.stdout = f print(resp.data) sys.stdout = original_stdout
The same applies when what arrives is json and you want to write it out nicely
import sys import json … with open("log.txt", "a") as f: sys.stdout = f print(json.dumps(resp.data, indent=4)) sys.stdout = original_stdout
An example of working with bytes
bmp.py
"""A module for dealing with BMP bitmap image files.""" def write_grayscale(filename, pixels): """Creates and writes a grayscale BMP file. Args: filename: The name of the BMP file to be created. pixels: A rectangular image stored as a sequence of rows. Each row must be an iterable series of integers in the range 0-255. Raises: ValueError: If any of the integer values are out of range. OSError: If the file couldn't be written. """ height = len(pixels) width = len(pixels[0]) with open(filename, 'wb') as bmp: # BMP Header bmp.write(b'BM') size_bookmark = bmp.tell() # The next four bytes hold the filesize as a 32-bit bmp.write(b'\x00\x00\x00\x00') # little-endian integer. Zero placeholder for now. bmp.write(b'\x00\x00') # Unused 16-bit integet - should be zero bmp.write(b'\x00\x00') # Unused 16-bit integet - should be zero pixel_offset_bookmark = bmp.tell() # The next four bytes hold the integer offset to the bmp.write(b'\x00\x00\x00\x00') # pixel data. Zero placeholder for now. # Image Header bmp.write(b'\x28\x00\x00\x00') # Image header size in bytes - 40 decimal bmp.write(_int32_to_bytes(width)) # Image width in pixels bmp.write(_int32_to_bytes(height)) # Image height in pixels bmp.write(b'\x01\x00') # Number of image planes bmp.write(b'\x08\x00') # Bits per pixel 8 for grayscale bmp.write(b'\x00\x00\x00\x00') # No compression bmp.write(b'\x00\x00\x00\x00') # Zero for uncompressed images bmp.write(b'\x00\x00\x00\x00') # Unused pixels per meter bmp.write(b'\x00\x00\x00\x00') # Unused pixels per meter bmp.write(b'\x00\x00\x00\x00') # Use whole color table bmp.write(b'\x00\x00\x00\x00') # All colors are important # Color palette - a linear grayscale for c in range(256): bmp.write(bytes((c, c, c, 0))) # Blue, Green, Red, Zero # Pixel data pixel_data_bookmark = bmp.tell() for row in reversed(pixels): # BMP files are bottom to top row_data = bytes(row) bmp.write(row_data) padding = b'\x00' * ((4 - (len(row) % 4)) % 4) # Pad row to multiple of four bytes bmp.write(padding) # End of file eof_bookmark = bmp.tell() # Fill in file size placeholder bmp.seek(size_bookmark) bmp.write(_int32_to_bytes(eof_bookmark)) # Fill in pixel offset placeholder bmp.seek(pixel_offset_bookmark) bmp.write(_int32_to_bytes(pixel_data_bookmark)) def _int32_to_bytes(i): """Convert an integer to four bytes in little-endian format.""" # &: Bitwise-and # >>: Right-shift return bytes( (i & 0xff, i >> 8 & 0xff, i >> 16 & 0xff, i >> 24 & 0xff) )
fractal.py
import math def mandel(real, imag): """The logarighm of number of iterations needed to determine whether a complex point is in the Mandelbrot set. Args: real: The real coordinate imag: The imaginary coordinate Returns: An integer in the range 1-255. """ x = 0 y = 0 for i in range(1, 257): if x*x + y*y > 4.0: break xt = real + x*x - y*y y = imag + 2.0 * x * y x = xt return int(math.log(i) * 256 / math.log(256)) -1 def mandelbrot(size_x, size_y): """Make an Mandelbrot set image. Args: size_x: Image width size_y: Image height Returns: A list of lists of integers in the range 0-255 """ return [[mandel((3.5 * x / size_x) - 2.5, (2.0 * y / size_y) - 1.0) for x in range(size_x)] for y in range(size_y)]
>>> import fractal >>> pixels = fractal.mandelbrot(448, 256) >>> import reprlib >>> reprlib.repr(pixels)
'[[31, 31, 31, 31, 31, 31, ...], [31, 31, 31, 31, 31, 31, ...], [31, 31, 31, 31, 31, 31, ...], [31, 31, 31, 31, 31, 31, ...], [31, 31, 31, 31, 31, 31, ...], [31, 31, 31, 31, 31, 31, ...], ...]'
>>> import bmp >>> bmp.write_grayscale("mandel.bmp", pixels)
Determining the dimensions of a bmp image
def dimensions(filename): """Determine the dimensions in pixels of a BMP image. Args: filename: The filename of a BMP file. Returns: A tuple containing two integers with the width and height in pixels. Raises: ValueError: If the file was not a BMP file. OSError: If there was a problem reading the file. """ with open(filename, 'rb') as f: magic = f.read(2) if magic != b'BM': raise ValueError(f"{filename} is not a BMP file") f.seek(18) width_bytes = f.read(4) height_bytes = f.read(4) return ( _bytes_to_int32(width_bytes), _bytes_to_int32(height_bytes)) def _bytes_to_int32(b): "Convert a bytes object containing four bytes into an integer." return b[0] | (b[1] << 8) | (b[2] << 16) | (b[3] << 24)
>>> import bmp >>> bmp.dimensions("mandel.bmp")
(448, 256)
Detect file encodings
An example script for detecting file encodings. To learn how to create files in different encodings in Linux read here
python -m pip install python-magic
import magic def get_encoding(sample): blob = open(sample, 'rb').read() m = magic.open(magic.MAGIC_MIME_ENCODING) m.load() encoding = m.buffer(blob) return encoding files = ['utf-8-file', 'windows-1251-file', 'shift-jis-file'] for f in files: print(get_encoding(f))
utf-8 iso-8859-1 unknown-8bit
There are still problems with detecting SHIFT-JIS
The path to a file
More about the library pathlib you can read here
python -m pip install pathlib
import pathlib from pathlib import Path dir_path = pathlib.Path.cwd() print(dir_path)
/home/andrei/sandbox/python/file_path
Read a file from another directory
Suppose we are in the directory one of the project file_path:
file_path/ ├── one │ └── path.py └── two └── sites.txt
cat ../two/sites.txt
www.heihei.website
To read the file sites.txt with Python, the library will help pathlib
import pathlib from pathlib import Path dir_path = pathlib.Path.cwd() path = Path(dir_path, "..", "two", "sites.txt") with open(path, "r") as f: sites = f.read() print(sites)
python path.py
Find a string
Let's look at the file sites.md and try to find the string heihei.website
topbicycle.ru heihei.website eht1.ru
We'll use the fact that the context manager returns an iterator and go through the file with the method next()
with open("sites.md", "r") as f: while True: try: line = str(next(f)) print(line) except StopIteration: raise ValueError("End of file") if line == "heihei.website\n": print("Found heihei.website!")
topbicycle.ru heihei.website Found heihei.website! eth1.ru Traceback (most recent call last): File "/mnt/c/Users/Andrei/readline_ex.py", line 4, in <module> line = str(next(f)) ^^^^^^^ StopIteration During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/mnt/c/Users/Andrei/readline_ex.py", line 7, in <module> raise ValueError("End of file") ValueError: End of file
With a for loop
with open("sites.md", "r") as f: for line in f: if line == "heihei.website\n": print("Found heihei.website") else: print("Some other line found")
Some other line found Found heihei.website Some other line found
Delete a specific line
Let's look at the file sites.md and try to delete all the lines heihei.website
topbicycle.ru heihei.website eht1.ru
with open("sites.md", "r") as f: with open("new_file.txt", "w") as nf: for line in f: if line != "heihei.website\n": nf.write(line) with open("new_file.txt", "r") as nf: print(nf.readlines())
['topbicycle.ru\n', 'eth1.ru\n']
Delete everything between two lines
Let's look at the file sites.md
topbicycle.ru heihei.website <bikes> Forward Stark Stels </bikes> eth1.ru devhops.ru
Let's delete everything between the <bikes> and </bikes> tags and write it to a new file.
Including the tags themselves:
copy = True with open("sites.md", "r") as f: print(f.readlines()) with open("sites.md", "r") as f: with open("clean.md", "w") as nf: for line in f: if line == "<bikes>\n": copy = False if copy: nf.write(line) if line == "</bikes>\n": copy = True with open("clean.md", "r") as nf: print(nf.readlines())
['topbicycle.ru\n', 'heihei.website\n', '<bikes>\n', 'Forward\n', 'Stark\n', 'Stels\n', '</bikes>\n', 'eth1.ru\n', 'devhops.ru\n'] ['topbicycle.ru\n', 'heihei.website\n', 'eth1.ru\n', 'devhops.ru\n']
Keeping the tags:
copy = True with open("sites.md", "r") as f: print(f.readlines()) with open("sites.md", "r") as f: with open("clean.md", "w") as nf: for line in f: if line == "</bikes>\n": copy = True if copy: nf.write(line) if line == "<bikes>\n": copy = False with open("clean.md", "r") as nf: print(nf.readlines())
['topbicycle.ru\n', 'heihei.website\n', '<bikes>\n', 'Forward\n', 'Stark\n', 'Stels\n', '</bikes>\n', 'eth1.ru\n', 'devhops.ru\n'] ['topbicycle.ru\n', 'heihei.website\n', '<bikes>\n', '</bikes>\n', 'eth1.ru\n', 'devhops.ru\n']
A more universal version using
functions
Here the changes will be written straight to the source file.
def main(): path = "sites.md" tag = "<bikes>" lines = readlines(path) delete_between_tags(path, lines, tag) def readlines(path): with open(path, "r") as f: lines = f.readlines() return lines def delete_between_tags(path, lines, tag): # or strip line.strip("\n") in a loop tag = str(tag) + "\n" taglist = list(tag) taglist.insert(1, "/") endtag = "".join(taglist) with open(path, "w") as f: copy = True for line in lines: if line == tag: copy = False if copy: f.write(line) if line == endtag: copy = True if __name__ == "__main__": main()
A tag does not have to be at the beginning of a line, so the script can be improved.
For example, in such a file the previous script will not delete anything.
heihei.website <topbicycle.ru> <bikes> Forward Stark Stels </bikes> </topbicycle.ru> eth1.ru devhops.ru
If we are sure that each tag is written on a separate line, it is enough to replace line = tag with tag in line and the same for endtag.
for line in lines: if tag in line: copy = False if copy: f.write(line) if endtag in line: copy = True
The line
tag = str(tag) + "\n"
also becomes unnecessary.
heihei.website <topbicycle.ru> </topbicycle.ru> eth1.ru devhops.ru
Append after a tag
Let's look at the file sites.md
heihei.website <topbicycle.ru> <bikes> Forward Stark Stels </bikes> </topbicycle.ru> eth1.ru devhops.ru
Let's append the following text after the <bikes> tag
<german> Cube Drossiger Ghost </german>
def main(): path = "sites.md" tag = "<bikes>" text = """ <german> Cube Drossiger Ghost </german> """ lines = readlines(path) append_after_tag(path, lines, tag, text) def readlines(path): with open(path, "r") as f: lines = f.readlines() return lines def append_after_tag(path, lines, tag, text): tag = str(tag) + "\n" taglist = list(tag) taglist.insert(1, "/") endtag = "".join(taglist) with open(path, "w") as f: write_flag = True for line in lines: f.write(line) if tag in line: if write_flag == True: f.write(text) write_flag = False if __name__ == "__main__": main()
heihei.website <topbicycle.ru> <bikes> <german> Cube Drossiger Ghost </german> Forward Stark Stels </bikes> </topbicycle.ru> eth1.ru devhops.ru
Remove the file extension
Let's look at the script remove_ex.py and the file openapi.yaml in the same directory. First let's look at using os.path.splitext()
demo `-- example |-- openapi.yaml `-- remove_ex.py
import os path = "openapi.yaml" print(os.path.splitext(path)[0])
We'll run the script from the current directory and from the parent one
python remove_ex.py
python example/remove_ex.py
openapi
Now suppose the file needs to be copied at the end of the script with shutil
import os import shutil path = "openapi.yaml" print(os.path.splitext(path)[0]) shutil.copyfile(path, "backup.yaml")
python remove_ex.py
openapi
A file will appear in the current directory backup.yaml
demo/ `-- example |-- backup.yaml |-- openapi.yaml `-- remove_ex.py
python example/remove_ex.py
FileNotFoundError: [Errno 2] No such file or directory: 'openapi.yaml'
When run from the current directory everything was fine, but from the parent directory shutil no longer finds the source file.
Let's use the full path to openapi.yaml
import os import shutil path = "openapi.yaml" path = os.path.join(os.path.dirname(__file__), path) print(os.path.splitext(path)[0]) shutil.copyfile(path, "backup.yaml")
From the current directory everything is still fine and backup.yaml is created in it as well.
python remove_ex.py
openapi
Nevertheless, already in this case when running from PyCharm the result will be
C:\Users\Andrei\demo\example\openapi openapi
When running from the parent directory
python example/remove_ex.py
example\openapi
backup.yaml is created in the parent directory
demo/ |-- backup.yaml `-- example |-- openapi.yaml `-- remove_ex.py
To extract only the file name, the following will help os.path.basename()
import os import shutil path = "openapi.yaml" path = os.path.join(os.path.dirname(__file__), path) print(os.path.splitext(os.path.basename(path))[0]) shutil.copyfile(path, "backup.yaml")
pathlib.Path().stem
Now let's look at using pathlib.Path().stem
demo `-- example |-- openapi.yaml `-- remove_ex.py
from pathlib import Path path = "openapi.yaml" print(Path(path).stem)
We'll run from the current directory and from the parent one
python remove_ex.py
python example/remove_ex.py
openapi
Now suppose the file needs to be copied at the end of the script with shutil
import shutil from pathlib import Path path = "openapi.yaml" print(Path(path).stem) shutil.copyfile(path, "backup.yaml")
python remove_ex.py
openapi
A file will appear in the current directory backup.yaml
demo/ `-- example |-- backup.yaml |-- openapi.yaml `-- remove_ex.py
python example/remove_ex.py
FileNotFoundError: [Errno 2] No such file or directory: 'openapi.yaml'
When run from the current directory everything was fine, but from the parent directory shutil no longer finds the source file.
Let's use the full path to openapi.yaml
import shutil from pathlib import Path path = "openapi.yaml" path = os.path.join(os.path.dirname(__file__), path) print(Path(path).stem) shutil.copyfile(path, "backup.yaml")
From the current directory everything is still fine and backup.yaml is created in it as well.
python remove_ex.py
openapi
When running from the parent directory
python example/remove_ex.py
openapi
backup.yaml is created in the parent directory
demo/ |-- backup.yaml `-- example |-- openapi.yaml `-- remove_ex.py
An example of a script that:
Converts a yaml file to json
Saves the original versions to the backup directory
Indicates the date and time of saving in the name of the backup files.
import yaml import json import time import shutil import os YAML_FILE_PATH = os.path.join(os.path.dirname(__file__), "openapi.yaml") JSON_FILE_PATH = os.path.join(os.path.dirname(__file__), "swagger.json") BACKUP_PATH = os.path.join(os.path.dirname(__file__), "backup") def yaml_to_json(yaml_file_path, json_file_path): with open(yaml_file_path, 'r') as yaml_file: yaml_data = yaml.safe_load(yaml_file) with open(json_file_path, 'w') as json_file: json.dump(yaml_data, json_file, indent=2) def backup(*args): for path in args: timestr = time.strftime("-%Y%m%d-%H%M%S") name = os.path.splitext(os.path.basename(path))[0] ext = os.path.splitext(os.path.basename(path))[1] name_with_time = name + timestr + ext dest = os.path.join(BACKUP_PATH, name_with_time) shutil.copyfile(path, dest) if __name__ == "__main__": if not os.path.exists(BACKUP_PATH): os.makedirs(BACKUP_PATH) backup(YAML_FILE_PATH, JSON_FILE_PATH) yaml_to_json(YAML_FILE_PATH, JSON_FILE_PATH)
Article author: Andrei Olegovich