Python 3.15: safe UTF-8 migration for Persian files

File: encoding_probe.py. Save this complete file without removing lines.

import json
import locale
import sys


report = {
    'python': sys.version.split()[0],
    'utf8_mode': bool(sys.flags.utf8_mode),
    'locale_encoding': locale.getencoding(),
    'filesystem_encoding': sys.getfilesystemencoding(),
    'target_default_changed_in': '3.15',
}
print(json.dumps(report, ensure_ascii=True))

python encoding_probe.py prints the current environment as JSON. To observe the 3.15 default, run it using that interpreter and the service configuration.

File: text_files.py. Save this complete file without removing lines.

import json
from pathlib import Path


def read_utf8(path):
    return Path(path).read_text(encoding='utf-8', errors='strict')


def write_report(path, value):
    with Path(path).open('x', encoding='utf-8', newline='\n') as output:
        json.dump(value, output, ensure_ascii=False, indent=2)
        output.write('\n')


def convert_legacy(source, destination, *, source_encoding, max_bytes=4 * 1024 * 1024):
    if max_bytes < 1:
        raise ValueError('max_bytes must be positive')
    source, destination = Path(source).resolve(), Path(destination).resolve()
    if source == destination:
        raise ValueError('Preserve the original source file')
    with source.open('rb') as original:
        raw = original.read(max_bytes + 1)
    if len(raw) > max_bytes:
        raise ValueError('Input exceeds the configured size limit')
    text = raw.decode(source_encoding, errors='strict')
    with destination.open('x', encoding='utf-8', newline='') as output:
        output.write(text)
    return destination

This example caps input at four megabytes. It preserves the source, requires a declared source encoding and creates destinations with exclusive mode.

File: test_text_files.py. Save this complete file without removing lines.

from pathlib import Path
import json
import tempfile
import unittest
from text_files import convert_legacy, read_utf8, write_report


class EncodingTests(unittest.TestCase):
    def test_persian_json_roundtrip(self):
        with tempfile.TemporaryDirectory(prefix='liyan-utf8-json-') as folder:
            path = Path(folder) / 'report.json'
            expected = {'name': 'لیان', 'message': 'سلام دنیا'}
            write_report(path, expected)
            self.assertEqual(json.loads(read_utf8(path)), expected)
            with self.assertRaises(FileExistsError):
                write_report(path, {'name': 'overwrite'})
            self.assertEqual(json.loads(read_utf8(path)), expected)

    def test_explicit_legacy_conversion(self):
        with tempfile.TemporaryDirectory(prefix='liyan-utf8-legacy-') as folder:
            old, new = Path(folder) / 'legacy.txt', Path(folder) / 'utf8.txt'
            old.write_bytes('سلام'.encode('cp1256'))
            convert_legacy(old, new, source_encoding='cp1256')
            self.assertEqual(read_utf8(new), 'سلام')
            self.assertNotEqual(old.read_bytes(), new.read_bytes())

    def test_decode_error_and_size_limit_do_not_create_output(self):
        with tempfile.TemporaryDirectory(prefix='liyan-utf8-failure-') as folder:
            old, new = Path(folder) / 'bad.txt', Path(folder) / 'new.txt'
            old.write_bytes(bytes([255]))
            with self.assertRaises(UnicodeDecodeError):
                convert_legacy(old, new, source_encoding='utf-8')
            self.assertFalse(new.exists())
            old.write_bytes(b'12345')
            with self.assertRaises(ValueError):
                convert_legacy(old, new, source_encoding='ascii', max_bytes=4)
            self.assertFalse(new.exists())


if __name__ == '__main__':
    unittest.main()

Run python -m unittest test_text_files.py. A decode error or oversized input must not create a destination, and an existing report must survive.

File: migrate_demo.py. Save this complete file without removing lines.

import json
from pathlib import Path
import tempfile
from text_files import convert_legacy, read_utf8, write_report


with tempfile.TemporaryDirectory(prefix='liyan-encoding-demo-') as folder:
    source = Path(folder) / 'legacy.txt'
    destination = Path(folder) / 'utf8.txt'
    source.write_bytes('سلام'.encode('cp1256'))
    convert_legacy(source, destination, source_encoding='cp1256')
    report = {'converted_text': read_utf8(destination), 'original_preserved': source.exists()}
    report_path = Path(folder) / 'report.json'
    write_report(report_path, report)
    assert json.loads(read_utf8(report_path)) == report
    print(json.dumps(report, ensure_ascii=True))

python migrate_demo.py converts a controlled CP1256 fixture to UTF-8 and reads its report back. Console JSON uses ASCII escapes so terminal rendering is not confused with file correctness.