From b884a9c3874cfe0eb885bf2b8eaa46a628a9f4a3 Mon Sep 17 00:00:00 2001 From: Andreas Weinzierl <20560137+aWeinzierl@users.noreply.github.com> Date: Wed, 27 Jan 2021 22:09:15 +0100 Subject: [PATCH 1/3] Set encoding to utf-8 when saving json file for stickerpack Fixes UnicodeEncodeError with Windows 10 when trying to import sticker packs caused by the default encoding scheme in Windows --- sticker/stickerimport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sticker/stickerimport.py b/sticker/stickerimport.py index 6846184..1bff240 100644 --- a/sticker/stickerimport.py +++ b/sticker/stickerimport.py @@ -99,7 +99,7 @@ async def reupload_pack(client: TelegramClient, pack: StickerSetFull, output_dir doc["body"] = sticker.emoticon doc["net.maunium.telegram.sticker"]["emoticons"].append(sticker.emoticon) - with open(pack_path, "w") as pack_file: + with open(pack_path, "w", encoding="utf-8") as pack_file: json.dump({ "title": pack.set.title, "id": f"tg-{pack.set.id}", From 0f7b678f5776a940b590e55381eba5ccb511d9d7 Mon Sep 17 00:00:00 2001 From: Andreas Weinzierl <20560137+aWeinzierl@users.noreply.github.com> Date: Wed, 27 Jan 2021 23:31:33 +0100 Subject: [PATCH 2/3] Use utf8-encoding whenever JSON is processed --- sticker/lib/util.py | 6 ++++-- sticker/pack.py | 6 +++--- sticker/scalar_convert.py | 8 ++++---- sticker/stickerimport.py | 4 ++-- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/sticker/lib/util.py b/sticker/lib/util.py index 240e8ea..2d58c82 100644 --- a/sticker/lib/util.py +++ b/sticker/lib/util.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from functools import partial from io import BytesIO import os.path import json @@ -21,6 +22,7 @@ from PIL import Image from . import matrix +open_utf8 = partial(open, encoding='UTF-8') def convert_image(data: bytes) -> (bytes, int, int): image: Image.Image = Image.open(BytesIO(data)).convert("RGBA") @@ -41,7 +43,7 @@ def convert_image(data: bytes) -> (bytes, int, int): def add_to_index(name: str, output_dir: str) -> None: index_path = os.path.join(output_dir, "index.json") try: - with open(index_path) as index_file: + with open_utf8(index_path) as index_file: index_data = json.load(index_file) except (FileNotFoundError, json.JSONDecodeError): index_data = {"packs": []} @@ -49,7 +51,7 @@ def add_to_index(name: str, output_dir: str) -> None: index_data["homeserver_url"] = matrix.homeserver_url if name not in index_data["packs"]: index_data["packs"].append(name) - with open(index_path, "w") as index_file: + with open_utf8(index_path, "w") as index_file: json.dump(index_data, index_file, indent=" ") print(f"Added {name} to {index_path}") diff --git a/sticker/pack.py b/sticker/pack.py index e611b73..f082370 100644 --- a/sticker/pack.py +++ b/sticker/pack.py @@ -93,7 +93,7 @@ async def main(args: argparse.Namespace) -> None: dirname = os.path.basename(os.path.abspath(args.path)) meta_path = os.path.join(args.path, "pack.json") try: - with open(meta_path) as pack_file: + with util.open_utf8(meta_path) as pack_file: pack = json.load(pack_file) print(f"Loaded existing pack meta from {meta_path}") except FileNotFoundError: @@ -112,14 +112,14 @@ async def main(args: argparse.Namespace) -> None: if sticker: pack["stickers"].append(sticker) - with open(meta_path, "w") as pack_file: + with util.open_utf8(meta_path, "w") as pack_file: json.dump(pack, pack_file) print(f"Wrote pack to {meta_path}") if args.add_to_index: picker_file_name = f"{pack['id']}.json" picker_pack_path = os.path.join(args.add_to_index, picker_file_name) - with open(picker_pack_path, "w") as pack_file: + with util.open_utf8(picker_pack_path, "w") as pack_file: json.dump(pack, pack_file) print(f"Copied pack to {picker_pack_path}") util.add_to_index(picker_file_name, args.add_to_index) diff --git a/sticker/scalar_convert.py b/sticker/scalar_convert.py index 69f7220..76a0101 100644 --- a/sticker/scalar_convert.py +++ b/sticker/scalar_convert.py @@ -19,12 +19,12 @@ import json index_path = "../web/packs/index.json" try: - with open(index_path) as index_file: + with util.open_utf8(index_path) as index_file: index_data = json.load(index_file) except (FileNotFoundError, json.JSONDecodeError): index_data = {"packs": []} -with open(sys.argv[-1]) as file: +with util.open_utf8(sys.argv[-1]) as file: data = json.load(file) for pack in data["assets"]: @@ -45,12 +45,12 @@ for pack in data["assets"]: } filename = f"scalar-{pack['name'].replace(' ', '_')}.json" pack_path = f"web/packs/{filename}" - with open(pack_path, "w") as pack_file: + with util.open_utf8(pack_path, "w") as pack_file: json.dump(pack_data, pack_file) print(f"Wrote {title} to {pack_path}") if filename not in index_data["packs"]: index_data["packs"].append(filename) -with open(index_path, "w") as index_file: +with util.open_utf8(index_path, "w") as index_file: json.dump(index_data, index_file, indent=" ") print(f"Updated {index_path}") diff --git a/sticker/stickerimport.py b/sticker/stickerimport.py index 1bff240..5327cc6 100644 --- a/sticker/stickerimport.py +++ b/sticker/stickerimport.py @@ -71,7 +71,7 @@ async def reupload_pack(client: TelegramClient, pack: StickerSetFull, output_dir already_uploaded = {} try: - with open(pack_path) as pack_file: + with util.open_utf8(pack_path, encoding="utf-8") as pack_file: existing_pack = json.load(pack_file) already_uploaded = {int(sticker["net.maunium.telegram.sticker"]["id"]): sticker for sticker in existing_pack["stickers"]} @@ -99,7 +99,7 @@ async def reupload_pack(client: TelegramClient, pack: StickerSetFull, output_dir doc["body"] = sticker.emoticon doc["net.maunium.telegram.sticker"]["emoticons"].append(sticker.emoticon) - with open(pack_path, "w", encoding="utf-8") as pack_file: + with util.open_utf8(pack_path, "w") as pack_file: json.dump({ "title": pack.set.title, "id": f"tg-{pack.set.id}", From 569d9815c65efe14e9ffba0899600a557269c7a6 Mon Sep 17 00:00:00 2001 From: Andreas Weinzierl <20560137+aWeinzierl@users.noreply.github.com> Date: Thu, 28 Jan 2021 00:06:58 +0100 Subject: [PATCH 3/3] remove redundant utf-8 setting --- sticker/stickerimport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sticker/stickerimport.py b/sticker/stickerimport.py index 5327cc6..c852e78 100644 --- a/sticker/stickerimport.py +++ b/sticker/stickerimport.py @@ -71,7 +71,7 @@ async def reupload_pack(client: TelegramClient, pack: StickerSetFull, output_dir already_uploaded = {} try: - with util.open_utf8(pack_path, encoding="utf-8") as pack_file: + with util.open_utf8(pack_path) as pack_file: existing_pack = json.load(pack_file) already_uploaded = {int(sticker["net.maunium.telegram.sticker"]["id"]): sticker for sticker in existing_pack["stickers"]}