From 97354bc0f96f79b8c8ed44795ce7cc6b5b69c5fa Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Mon, 9 Oct 2023 17:23:05 +0800 Subject: [PATCH 01/25] Add: support for Misskey notes --- nazurin/sites/misskey/__init__.py | 0 nazurin/sites/misskey/api.py | 134 +++++++++++++++++++++++++++++ nazurin/sites/misskey/commands.py | 0 nazurin/sites/misskey/config.py | 10 +++ nazurin/sites/misskey/interface.py | 23 +++++ 5 files changed, 167 insertions(+) create mode 100644 nazurin/sites/misskey/__init__.py create mode 100644 nazurin/sites/misskey/api.py create mode 100644 nazurin/sites/misskey/commands.py create mode 100644 nazurin/sites/misskey/config.py create mode 100644 nazurin/sites/misskey/interface.py diff --git a/nazurin/sites/misskey/__init__.py b/nazurin/sites/misskey/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py new file mode 100644 index 00000000..cfc8e7d4 --- /dev/null +++ b/nazurin/sites/misskey/api.py @@ -0,0 +1,134 @@ +import os +import datetime +from pathlib import Path +import shlex +import subprocess +from types import dict, Tuple + +from nazurin.models import Caption, Illust, Image, Ugoira +from nazurin.models.file import File +from nazurin.utils import Request, logger +from nazurin.utils.decorators import network_retry, async_wrap +from nazurin.utils.exceptions import NazurinError + +from .config import DESTINATION, FILENAME + + +class Misskey: + @network_retry + async def get_note(self, site_url: str, note_id: str) -> dict: + """Fetch a note from centain site's API.""" + api = f"https://{site_url}/api/notes/show" + data = { + "noteId": note_id + } + + async with Request() as request: + async with request.post(url=api, data=data) as response: + if response.status == 400: + raise NazurinError("Note not found") + response.raise_for_status() + + data = await response.json() + if "error" in data: + logger.error(data) + raise NazurinError(data["error"]) + + return data + + def build_caption(self, note: dict, site_url: str) -> Caption: + return Caption( + { + "url": f"https://{site_url}/notes/{note['id']}", + "ori_url": note["uri"], + "author": f"{note['user']['username']} #{note['user']['name']}", + "text": note["text"], + } + ) + + async def get_video(self, file: dict, destination: str, filename: str) -> File: + if file["type"] != "video/mp4": + @async_wrap + def convert(config: File, output: File): + config_path = Path(config.path).as_posix() + # Copy video and audio streams + args = [ + "ffmpeg", + "-i", + config_path, + "-vcodec", + "copy", + "-acodec", + "copy", + output.path, + ] + cmd = shlex.join(args) + logger.info("Calling FFmpeg with command: {}", cmd) + try: + output = subprocess.check_output( + args, stderr=subprocess.STDOUT, shell=False + ) + except subprocess.CalledProcessError as error: + logger.error( + "FFmpeg failed with code {}, output:\n {}", + error.returncode, + error.output.decode(), + ) + raise NazurinError("Failed to convert ugoira to mp4.") from None + + ori_video = File(filename, file["url"]) + await ori_video.download() + filename, _ = os.path.splitext(filename) + video = File(filename + ".mp4", None, destination) + await convert(ori_video, video) + else: + video = File(filename, file["url"], destination) + return video + + def parse_note(self, note: dict, site_url: str) -> Illust: + """Get images and build caption.""" + images = [] + files = [] + files = note["files"] + for file in files: + destination, filename = self.get_storage_dest(note, file["name"]) + if file["type"].startswith("image"): + images.append( + Image( + filename, + file["url"], + destination, + file["thumbnailUrl"], + file["size"], + file["properties"]["width"], + file["properties"]["height"], + ) + ) + elif file["type"].startswith("video"): + files.append(self.get_video(file, destination, filename)) + # Build note caption + caption = self.build_caption(note, site_url) + return Illust(images, caption, note, files) + + async def fetch(self, site_url: str, post_id: str) -> Illust: + note = await self.get_note(site_url, post_id) + return self.parse_note(note, site_url) + + @staticmethod + def get_storage_dest(note: dict, filename: str) -> Tuple[str, str]: + """ + Format destination and filename. + """ + created_at = datetime.fromisoformat(note["createdAt"]) + filename, extension = os.path.splitext(filename) + context = { + **note, + # Human-friendly filename, without extension + "filename": filename, + "created_at": created_at, + "extension": extension, + } + return ( + DESTINATION.format_map(context), + FILENAME.format_map(context) + extension, + ) diff --git a/nazurin/sites/misskey/commands.py b/nazurin/sites/misskey/commands.py new file mode 100644 index 00000000..e69de29b diff --git a/nazurin/sites/misskey/config.py b/nazurin/sites/misskey/config.py new file mode 100644 index 00000000..38c04f4f --- /dev/null +++ b/nazurin/sites/misskey/config.py @@ -0,0 +1,10 @@ +from nazurin.config import env + +PRIORITY = 30 +COLLECTION = "misskey" + +with env.prefixed("MISSKEY_"): + with env.prefixed("FILE_"): + DESTINATION: str = env.str("PATH", default="Misskey") + FILENAME: str = env.str( + "NAME", default="{filename} - {user[name]}({user[username]})") \ No newline at end of file diff --git a/nazurin/sites/misskey/interface.py b/nazurin/sites/misskey/interface.py new file mode 100644 index 00000000..5b574392 --- /dev/null +++ b/nazurin/sites/misskey/interface.py @@ -0,0 +1,23 @@ +from time import time + +from nazurin.database import Database +from nazurin.models import Illust + +from .api import Misskey +from .config import COLLECTION + +patterns = [ + # https://site.example/notes/9khcu788zb + r"://(.*?)/notes/(.*)", +] + +async def handle(match) -> Illust: + site_url = match.group(1) + post_id = match.group(2) + db = Database().driver() + collection = db.collection(COLLECTION) + + illust = await Misskey().fetch(site_url, post_id) + illust.metadata["collected_at"] = time() + await collection.insert(int(post_id), illust.metadata) + return illust From 0126d4171f2a994a7ea01ac145d6cc2524678265 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Wed, 25 Oct 2023 13:47:18 +0800 Subject: [PATCH 02/25] Fix: lint error --- nazurin/sites/misskey/api.py | 46 ++++++++++++++++-------------- nazurin/sites/misskey/config.py | 2 +- nazurin/sites/misskey/interface.py | 3 +- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index cfc8e7d4..795f95be 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -1,11 +1,11 @@ import os -import datetime +from datetime import datetime from pathlib import Path import shlex import subprocess -from types import dict, Tuple +from typing import Tuple -from nazurin.models import Caption, Illust, Image, Ugoira +from nazurin.models import Caption, Illust, Image from nazurin.models.file import File from nazurin.utils import Request, logger from nazurin.utils.decorators import network_retry, async_wrap @@ -22,30 +22,30 @@ async def get_note(self, site_url: str, note_id: str) -> dict: data = { "noteId": note_id } - + async with Request() as request: async with request.post(url=api, data=data) as response: if response.status == 400: raise NazurinError("Note not found") response.raise_for_status() - + data = await response.json() if "error" in data: logger.error(data) raise NazurinError(data["error"]) - + return data - + def build_caption(self, note: dict, site_url: str) -> Caption: return Caption( - { - "url": f"https://{site_url}/notes/{note['id']}", - "ori_url": note["uri"], - "author": f"{note['user']['username']} #{note['user']['name']}", - "text": note["text"], - } - ) - + { + "url": f"https://{site_url}/notes/{note['id']}", + "ori_url": note["uri"], + "author": f"{note['user']['username']} #{note['user']['name']}", + "text": note["text"], + } + ) + async def get_video(self, file: dict, destination: str, filename: str) -> File: if file["type"] != "video/mp4": @async_wrap @@ -74,17 +74,19 @@ def convert(config: File, output: File): error.returncode, error.output.decode(), ) - raise NazurinError("Failed to convert ugoira to mp4.") from None + raise NazurinError( + "Failed to convert ugoira to mp4.") from None ori_video = File(filename, file["url"]) - await ori_video.download() + async with Request() as session: + await ori_video.download(session) filename, _ = os.path.splitext(filename) - video = File(filename + ".mp4", None, destination) - await convert(ori_video, video) + video = File(filename + ".mp4", "", destination) + convert(ori_video, video) else: video = File(filename, file["url"], destination) return video - + def parse_note(self, note: dict, site_url: str) -> Illust: """Get images and build caption.""" images = [] @@ -109,11 +111,11 @@ def parse_note(self, note: dict, site_url: str) -> Illust: # Build note caption caption = self.build_caption(note, site_url) return Illust(images, caption, note, files) - + async def fetch(self, site_url: str, post_id: str) -> Illust: note = await self.get_note(site_url, post_id) return self.parse_note(note, site_url) - + @staticmethod def get_storage_dest(note: dict, filename: str) -> Tuple[str, str]: """ diff --git a/nazurin/sites/misskey/config.py b/nazurin/sites/misskey/config.py index 38c04f4f..06c915f5 100644 --- a/nazurin/sites/misskey/config.py +++ b/nazurin/sites/misskey/config.py @@ -7,4 +7,4 @@ with env.prefixed("FILE_"): DESTINATION: str = env.str("PATH", default="Misskey") FILENAME: str = env.str( - "NAME", default="{filename} - {user[name]}({user[username]})") \ No newline at end of file + "NAME", default="{filename} - {user[name]}({user[username]})") diff --git a/nazurin/sites/misskey/interface.py b/nazurin/sites/misskey/interface.py index 5b574392..16e725b0 100644 --- a/nazurin/sites/misskey/interface.py +++ b/nazurin/sites/misskey/interface.py @@ -11,12 +11,13 @@ r"://(.*?)/notes/(.*)", ] + async def handle(match) -> Illust: site_url = match.group(1) post_id = match.group(2) db = Database().driver() collection = db.collection(COLLECTION) - + illust = await Misskey().fetch(site_url, post_id) illust.metadata["collected_at"] = time() await collection.insert(int(post_id), illust.metadata) From cc26d9ccae43f64c2a2699e4b76e8a193499ed96 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Wed, 25 Oct 2023 14:52:13 +0800 Subject: [PATCH 03/25] Update: remove commands for Misskey --- nazurin/sites/misskey/commands.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 nazurin/sites/misskey/commands.py diff --git a/nazurin/sites/misskey/commands.py b/nazurin/sites/misskey/commands.py deleted file mode 100644 index e69de29b..00000000 From 0d923f5d6f6df46907d86e70390ac5f2a46c693e Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Thu, 26 Oct 2023 10:09:27 +0800 Subject: [PATCH 04/25] Add: __init__.py --- nazurin/sites/misskey/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nazurin/sites/misskey/__init__.py b/nazurin/sites/misskey/__init__.py index e69de29b..0bf129f2 100644 --- a/nazurin/sites/misskey/__init__.py +++ b/nazurin/sites/misskey/__init__.py @@ -0,0 +1,6 @@ +"""Misskey site plugin""" +from .api import Misskey +from .config import PRIORITY +from .interface import handle, patterns + +__all__ = ["Misskey", "PRIORITY", "patterns", "handle"] From 1ed31f09c22415ade211657727b9114b3699c0e5 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Thu, 26 Oct 2023 10:27:45 +0800 Subject: [PATCH 05/25] Fix: wrong JSON post args issue --- nazurin/sites/misskey/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index 795f95be..78652e75 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -19,12 +19,12 @@ class Misskey: async def get_note(self, site_url: str, note_id: str) -> dict: """Fetch a note from centain site's API.""" api = f"https://{site_url}/api/notes/show" - data = { + json = { "noteId": note_id } async with Request() as request: - async with request.post(url=api, data=data) as response: + async with request.post(url=api, json=json) as response: if response.status == 400: raise NazurinError("Note not found") response.raise_for_status() From 853781d2fe0a83a546e40987b23ee5c9decfe7b4 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Thu, 26 Oct 2023 10:35:21 +0800 Subject: [PATCH 06/25] Fix: datetime.fromisoformat format error --- nazurin/sites/misskey/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index 78652e75..e943d55a 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -121,7 +121,8 @@ def get_storage_dest(note: dict, filename: str) -> Tuple[str, str]: """ Format destination and filename. """ - created_at = datetime.fromisoformat(note["createdAt"]) + # remove 'Z' to fit datetime.fromisoformat's needs + created_at = datetime.fromisoformat(note["createdAt"][:-1]) filename, extension = os.path.splitext(filename) context = { **note, From 6c2451b6549fa6ff62a82085ea8fabdfcf761eee Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Thu, 26 Oct 2023 11:09:30 +0800 Subject: [PATCH 07/25] Fix: interface - post_id wrong type issue --- nazurin/sites/misskey/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nazurin/sites/misskey/interface.py b/nazurin/sites/misskey/interface.py index 16e725b0..5abd74a9 100644 --- a/nazurin/sites/misskey/interface.py +++ b/nazurin/sites/misskey/interface.py @@ -20,5 +20,5 @@ async def handle(match) -> Illust: illust = await Misskey().fetch(site_url, post_id) illust.metadata["collected_at"] = time() - await collection.insert(int(post_id), illust.metadata) + await collection.insert(str(post_id), illust.metadata) return illust From 6f314202b9199b596802b859d4c998f24f6a0734 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Mon, 30 Oct 2023 11:20:30 +0800 Subject: [PATCH 08/25] Fix: file "url" missing issue --- nazurin/sites/misskey/api.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index e943d55a..060952be 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -3,7 +3,7 @@ from pathlib import Path import shlex import subprocess -from typing import Tuple +from typing import List, Tuple from nazurin.models import Caption, Illust, Image from nazurin.models.file import File @@ -89,10 +89,10 @@ def convert(config: File, output: File): def parse_note(self, note: dict, site_url: str) -> Illust: """Get images and build caption.""" - images = [] - files = [] - files = note["files"] - for file in files: + images: List[Image] = [] + files: List[File] = [] + file_dict = note["files"] + for file in file_dict: destination, filename = self.get_storage_dest(note, file["name"]) if file["type"].startswith("image"): images.append( From b7a89fe3f946a79541fd84975c7b45780d68d445 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Mon, 30 Oct 2023 14:15:18 +0800 Subject: [PATCH 09/25] Fix: misskey note parsing issue - coroutine issues - local instance & remote instance has different res json --- nazurin/sites/misskey/api.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index 060952be..64c4230c 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -37,10 +37,11 @@ async def get_note(self, site_url: str, note_id: str) -> dict: return data def build_caption(self, note: dict, site_url: str) -> Caption: + url = f"https://{site_url}/notes/{note['id']}" return Caption( { - "url": f"https://{site_url}/notes/{note['id']}", - "ori_url": note["uri"], + "url": url, + "ori_url": note.get("uri", url), "author": f"{note['user']['username']} #{note['user']['name']}", "text": note["text"], } @@ -82,12 +83,12 @@ def convert(config: File, output: File): await ori_video.download(session) filename, _ = os.path.splitext(filename) video = File(filename + ".mp4", "", destination) - convert(ori_video, video) + await convert(ori_video, video) else: video = File(filename, file["url"], destination) return video - def parse_note(self, note: dict, site_url: str) -> Illust: + async def parse_note(self, note: dict, site_url: str) -> Illust: """Get images and build caption.""" images: List[Image] = [] files: List[File] = [] @@ -107,14 +108,14 @@ def parse_note(self, note: dict, site_url: str) -> Illust: ) ) elif file["type"].startswith("video"): - files.append(self.get_video(file, destination, filename)) + files.append(await self.get_video(file, destination, filename)) # Build note caption caption = self.build_caption(note, site_url) return Illust(images, caption, note, files) async def fetch(self, site_url: str, post_id: str) -> Illust: note = await self.get_note(site_url, post_id) - return self.parse_note(note, site_url) + return await self.parse_note(note, site_url) @staticmethod def get_storage_dest(note: dict, filename: str) -> Tuple[str, str]: From 490d863c3e62d9868f0209a90433ae07fcd6e5e0 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Mon, 30 Oct 2023 14:24:03 +0800 Subject: [PATCH 10/25] Fix: ffmpeg overwrite issue --- nazurin/sites/misskey/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index 64c4230c..a7558f93 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -61,6 +61,7 @@ def convert(config: File, output: File): "copy", "-acodec", "copy", + "-y", output.path, ] cmd = shlex.join(args) From e9f4290929a07041f9d544fdcf9a0e3ec1fb8550 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Tue, 31 Oct 2023 16:26:42 +0800 Subject: [PATCH 11/25] Add: docs for Misskey site --- docs/includes/site.md | 1 + docs/includes/site.zh.md | 1 + docs/site/misskey.md | 57 ++++++++++++++++++++++++++++++++++++++++ docs/site/misskey_zh.md | 57 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+) create mode 100644 docs/site/misskey.md create mode 100644 docs/site/misskey_zh.md diff --git a/docs/includes/site.md b/docs/includes/site.md index a589c35c..2e1ff898 100644 --- a/docs/includes/site.md +++ b/docs/includes/site.md @@ -15,3 +15,4 @@ | Weibo | | | ✔ | | DeviantArt | | | ✔ | | Lofter | | | ✔ | +| Misskey | Various Sites | | ✔ | diff --git a/docs/includes/site.zh.md b/docs/includes/site.zh.md index af15327c..a2620109 100644 --- a/docs/includes/site.zh.md +++ b/docs/includes/site.zh.md @@ -15,3 +15,4 @@ | 微博 | | | ✔ | | DeviantArt | | | ✔ | | Lofter | | | ✔ | +| Misskey | 许多实例 | | ✔ | diff --git a/docs/site/misskey.md b/docs/site/misskey.md new file mode 100644 index 00000000..25cdd332 --- /dev/null +++ b/docs/site/misskey.md @@ -0,0 +1,57 @@ +# Misskey Notes + +From any Misskey instance. + +## Customizing Storage Path & File Name + +For more information, refer to [Customizing Storage Path & File Name](./index.md/#customizing-storage-path--file-name). + +### MISSKEY_FILE_PATH + +:material-lightbulb-on: Optional, defaults to `Misskey` + +Storage path for downloaded images. + +### MISSKEY_FILE_NAME + +:material-lightbulb-on: Optional, defaults to `{filename} - {user[name]}({user[username]})` + +File name for downloaded images. + +### Available Variables + +_Only common used ones are listed._ + +```json +{ + "id": "", + "createdAt": "2023-10-05T23:10:13.016Z", + "userId": "", + "user": { + "id": "", + "name": "", + "username": "", + "host": "" + }, + "text": "", + "fileIds": [""], + "files": [ + { + "id": "", + "createdAt": "2023-10-05T23:10:16.445Z", + "name": "", + "type": "image/webp", + "md5": "", + "size": 800000, + "isSensitive": false, + "properties": { + "width": 2048, + "height": 1969 + }, + "url": "", + "thumbnailUrl": "" + } + ], + "uri": "" // Only available when the note is from a remote instance. +} +``` diff --git a/docs/site/misskey_zh.md b/docs/site/misskey_zh.md new file mode 100644 index 00000000..cb5b544f --- /dev/null +++ b/docs/site/misskey_zh.md @@ -0,0 +1,57 @@ +# Misskey Notes + +从任何 Misskey 实例获取。 + +## 自定义存储路径和文件名 + +更多信息请查阅 [自定义存储路径和文件名](./index.zh.md/#customizing-storage-path--file-name)。 + +### Misskey_FILE_PATH + +:material-lightbulb-on: 可选,默认为 `Misskey` + +存储路径。 + +### Misskey_FILE_NAME + +:material-lightbulb-on: 可选,默认为 `{filename} - {user[name]}({user[username]})` + +文件名称。 + +### 可用变量 + +_此处只列出常用项。_ + +```json +{ + "id": "", + "createdAt": "2023-10-05T23:10:13.016Z", + "userId": "<用户 id>", + "user": { + "id": "<用户 id>", + "name": "<用户展示名称>", + "username": "<用户名>", + "host": "<用户所在实例 URL>" + }, + "text": "", + "fileIds": ["<文件 id>"], + "files": [ + { + "id": "<文件 id>", + "createdAt": "2023-10-05T23:10:16.445Z", + "name": "<文件名>", + "type": "image/webp", + "md5": "<文件 md5>", + "size": 800000, + "isSensitive": false, + "properties": { + "width": 2048, + "height": 1969 + }, + "url": "<文件 URL>", + "thumbnailUrl": "<缩略图 URL>" + } + ], + "uri": "<该 note 在源实例的 URL>" // Only available when the note is from a remote instance. +} +``` From 68f83c221ed26503a3a346efc96120e8639a9572 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Sun, 19 Nov 2023 07:18:41 +0000 Subject: [PATCH 12/25] Fix: GIF sending issue --- nazurin/sites/misskey/api.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index a7558f93..f22c3cdf 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -5,7 +5,7 @@ import subprocess from typing import List, Tuple -from nazurin.models import Caption, Illust, Image +from nazurin.models import Caption, Illust, Image, Ugoira from nazurin.models.file import File from nazurin.utils import Request, logger from nazurin.utils.decorators import network_retry, async_wrap @@ -48,7 +48,9 @@ def build_caption(self, note: dict, site_url: str) -> Caption: ) async def get_video(self, file: dict, destination: str, filename: str) -> File: - if file["type"] != "video/mp4": + if file["type"] == "video/mp4" or file["type"] == "image/gif": + video = File(filename, file["url"], destination) + else: @async_wrap def convert(config: File, output: File): config_path = Path(config.path).as_posix() @@ -85,18 +87,19 @@ def convert(config: File, output: File): filename, _ = os.path.splitext(filename) video = File(filename + ".mp4", "", destination) await convert(ori_video, video) - else: - video = File(filename, file["url"], destination) return video async def parse_note(self, note: dict, site_url: str) -> Illust: - """Get images and build caption.""" + """Build caption and get images.""" + # Build note caption + caption = self.build_caption(note, site_url) + images: List[Image] = [] files: List[File] = [] file_dict = note["files"] for file in file_dict: destination, filename = self.get_storage_dest(note, file["name"]) - if file["type"].startswith("image"): + if file["type"].startswith("image") and not file["type"].endswith("gif"): images.append( Image( filename, @@ -108,10 +111,9 @@ async def parse_note(self, note: dict, site_url: str) -> Illust: file["properties"]["height"], ) ) - elif file["type"].startswith("video"): - files.append(await self.get_video(file, destination, filename)) - # Build note caption - caption = self.build_caption(note, site_url) + elif file["type"].startswith("video") or file["type"].endswith("gif"): + return Ugoira(await self.get_video(file, destination, filename), caption, note) + return Illust(images, caption, note, files) async def fetch(self, site_url: str, post_id: str) -> Illust: From 67491affd1d9325ea220a7cb4de1be5f1f3bb9a5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 19 Nov 2023 07:47:23 +0000 Subject: [PATCH 13/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nazurin/sites/misskey/api.py | 18 +++++++++--------- nazurin/sites/misskey/config.py | 3 ++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index f22c3cdf..26908be2 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -1,14 +1,14 @@ import os -from datetime import datetime -from pathlib import Path import shlex import subprocess +from datetime import datetime +from pathlib import Path from typing import List, Tuple from nazurin.models import Caption, Illust, Image, Ugoira from nazurin.models.file import File from nazurin.utils import Request, logger -from nazurin.utils.decorators import network_retry, async_wrap +from nazurin.utils.decorators import async_wrap, network_retry from nazurin.utils.exceptions import NazurinError from .config import DESTINATION, FILENAME @@ -19,9 +19,7 @@ class Misskey: async def get_note(self, site_url: str, note_id: str) -> dict: """Fetch a note from centain site's API.""" api = f"https://{site_url}/api/notes/show" - json = { - "noteId": note_id - } + json = {"noteId": note_id} async with Request() as request: async with request.post(url=api, json=json) as response: @@ -51,6 +49,7 @@ async def get_video(self, file: dict, destination: str, filename: str) -> File: if file["type"] == "video/mp4" or file["type"] == "image/gif": video = File(filename, file["url"], destination) else: + @async_wrap def convert(config: File, output: File): config_path = Path(config.path).as_posix() @@ -78,8 +77,7 @@ def convert(config: File, output: File): error.returncode, error.output.decode(), ) - raise NazurinError( - "Failed to convert ugoira to mp4.") from None + raise NazurinError("Failed to convert ugoira to mp4.") from None ori_video = File(filename, file["url"]) async with Request() as session: @@ -112,7 +110,9 @@ async def parse_note(self, note: dict, site_url: str) -> Illust: ) ) elif file["type"].startswith("video") or file["type"].endswith("gif"): - return Ugoira(await self.get_video(file, destination, filename), caption, note) + return Ugoira( + await self.get_video(file, destination, filename), caption, note + ) return Illust(images, caption, note, files) diff --git a/nazurin/sites/misskey/config.py b/nazurin/sites/misskey/config.py index 06c915f5..3d5ef1d0 100644 --- a/nazurin/sites/misskey/config.py +++ b/nazurin/sites/misskey/config.py @@ -7,4 +7,5 @@ with env.prefixed("FILE_"): DESTINATION: str = env.str("PATH", default="Misskey") FILENAME: str = env.str( - "NAME", default="{filename} - {user[name]}({user[username]})") + "NAME", default="{filename} - {user[name]}({user[username]})" + ) From 29d4d3503eca16088abcdc187ff160af38132553 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Mon, 20 Nov 2023 02:17:54 +0000 Subject: [PATCH 14/25] Fix: Handle ClientResponseError in Misskey API --- nazurin/sites/misskey/api.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index f22c3cdf..5783e9e4 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -5,6 +5,7 @@ import subprocess from typing import List, Tuple +from aiohttp.client_exceptions import ClientResponseError from nazurin.models import Caption, Illust, Image, Ugoira from nazurin.models.file import File from nazurin.utils import Request, logger @@ -25,15 +26,12 @@ async def get_note(self, site_url: str, note_id: str) -> dict: async with Request() as request: async with request.post(url=api, json=json) as response: - if response.status == 400: - raise NazurinError("Note not found") - response.raise_for_status() + try: + response.raise_for_status() + except ClientResponseError as err: + raise NazurinError(err) from None data = await response.json() - if "error" in data: - logger.error(data) - raise NazurinError(data["error"]) - return data def build_caption(self, note: dict, site_url: str) -> Caption: From c573f8f7875086d9283daac5618d3de85760ead8 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Mon, 20 Nov 2023 02:27:37 +0000 Subject: [PATCH 15/25] fix: Refactor build_caption to handle note URI --- nazurin/sites/misskey/api.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index 5783e9e4..52b3ff52 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -36,14 +36,24 @@ async def get_note(self, site_url: str, note_id: str) -> dict: def build_caption(self, note: dict, site_url: str) -> Caption: url = f"https://{site_url}/notes/{note['id']}" - return Caption( - { - "url": url, - "ori_url": note.get("uri", url), - "author": f"{note['user']['username']} #{note['user']['name']}", - "text": note["text"], - } - ) + # URL from the original instance + if note["uri"] is None: + return Caption( + { + "url": url, + "author": f"{note['user']['username']} #{note['user']['name']}", + "text": note["text"], + } + ) + else: + return Caption( + { + "url": url, + "original_url": note["uri"], + "author": f"{note['user']['username']} #{note['user']['name']}", + "text": note["text"], + } + ) async def get_video(self, file: dict, destination: str, filename: str) -> File: if file["type"] == "video/mp4" or file["type"] == "image/gif": From 6e3f89f610e5fe5b82ba908196358173751a9c3d Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Mon, 20 Nov 2023 02:40:00 +0000 Subject: [PATCH 16/25] fix: Update Misskey file name template --- docs/includes/site.md | 2 +- docs/includes/site.zh.md | 2 +- docs/site/misskey.md | 2 +- docs/site/misskey_zh.md | 2 +- nazurin/sites/misskey/config.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/includes/site.md b/docs/includes/site.md index 2e1ff898..37ba06e6 100644 --- a/docs/includes/site.md +++ b/docs/includes/site.md @@ -15,4 +15,4 @@ | Weibo | | | ✔ | | DeviantArt | | | ✔ | | Lofter | | | ✔ | -| Misskey | Various Sites | | ✔ | +| Misskey | Any Misskey instance | | ✔ | diff --git a/docs/includes/site.zh.md b/docs/includes/site.zh.md index a2620109..2f41ce2b 100644 --- a/docs/includes/site.zh.md +++ b/docs/includes/site.zh.md @@ -15,4 +15,4 @@ | 微博 | | | ✔ | | DeviantArt | | | ✔ | | Lofter | | | ✔ | -| Misskey | 许多实例 | | ✔ | +| Misskey | 各种 Misskey 实例 | | ✔ | diff --git a/docs/site/misskey.md b/docs/site/misskey.md index 25cdd332..fe188aaf 100644 --- a/docs/site/misskey.md +++ b/docs/site/misskey.md @@ -14,7 +14,7 @@ Storage path for downloaded images. ### MISSKEY_FILE_NAME -:material-lightbulb-on: Optional, defaults to `{filename} - {user[name]}({user[username]})` +:material-lightbulb-on: Optional, defaults to `{id} - {filename} - {user[name]}({user[username]})` File name for downloaded images. diff --git a/docs/site/misskey_zh.md b/docs/site/misskey_zh.md index cb5b544f..2cc32658 100644 --- a/docs/site/misskey_zh.md +++ b/docs/site/misskey_zh.md @@ -14,7 +14,7 @@ ### Misskey_FILE_NAME -:material-lightbulb-on: 可选,默认为 `{filename} - {user[name]}({user[username]})` +:material-lightbulb-on: 可选,默认为 `{id} - {filename} - {user[name]}({user[username]})` 文件名称。 diff --git a/nazurin/sites/misskey/config.py b/nazurin/sites/misskey/config.py index 06c915f5..6a2a99b4 100644 --- a/nazurin/sites/misskey/config.py +++ b/nazurin/sites/misskey/config.py @@ -7,4 +7,4 @@ with env.prefixed("FILE_"): DESTINATION: str = env.str("PATH", default="Misskey") FILENAME: str = env.str( - "NAME", default="{filename} - {user[name]}({user[username]})") + "NAME", default="{id} - {filename} - {user[name]}({user[username]})") From 69b67e8a0867bd017d8bc83a6d190e8ea2a6b854 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Tue, 21 Nov 2023 01:47:05 +0000 Subject: [PATCH 17/25] add: JSON format validation in Misskey API --- nazurin/sites/misskey/api.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index 52b3ff52..b1cd2b26 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -16,9 +16,27 @@ class Misskey: + def check_res_json(self, data: dict) -> bool: + note_required_fields = ["id", "user", "text", "createdAt", "files"] + for f in note_required_fields: + if f not in data: + return False + user = data["user"] + user_required_fileds = ["username", "name"] + for f in user_required_fileds: + if f not in user: + return False + files = data["files"] + file_required_fields = ["name", "type", "url", "thumbnailUrl", "size", "properties"] + for file in files: + for f in file_required_fields: + if f not in file: + return False + return True + @network_retry async def get_note(self, site_url: str, note_id: str) -> dict: - """Fetch a note from centain site's API.""" + """Fetch a note from a Misskey instance.""" api = f"https://{site_url}/api/notes/show" json = { "noteId": note_id @@ -30,8 +48,12 @@ async def get_note(self, site_url: str, note_id: str) -> dict: response.raise_for_status() except ClientResponseError as err: raise NazurinError(err) from None - data = await response.json() + + # check JSON format + if not self.check_res_json(data): + raise NazurinError("Invalid JSON format.") + return data def build_caption(self, note: dict, site_url: str) -> Caption: From 0ae14479a41919a12b78474a9d8a7de0203f9c7c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Nov 2023 01:51:24 +0000 Subject: [PATCH 18/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nazurin/sites/misskey/api.py | 16 ++++++++++++---- nazurin/sites/misskey/config.py | 3 ++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index 89ae8909..432c9dfc 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -6,6 +6,7 @@ from typing import List, Tuple from aiohttp.client_exceptions import ClientResponseError + from nazurin.models import Caption, Illust, Image, Ugoira from nazurin.models.file import File from nazurin.utils import Request, logger @@ -27,13 +28,20 @@ def check_res_json(self, data: dict) -> bool: if f not in user: return False files = data["files"] - file_required_fields = ["name", "type", "url", "thumbnailUrl", "size", "properties"] + file_required_fields = [ + "name", + "type", + "url", + "thumbnailUrl", + "size", + "properties", + ] for file in files: for f in file_required_fields: if f not in file: return False return True - + @network_retry async def get_note(self, site_url: str, note_id: str) -> dict: """Fetch a note from a Misskey instance.""" @@ -47,11 +55,11 @@ async def get_note(self, site_url: str, note_id: str) -> dict: except ClientResponseError as err: raise NazurinError(err) from None data = await response.json() - + # check JSON format if not self.check_res_json(data): raise NazurinError("Invalid JSON format.") - + return data def build_caption(self, note: dict, site_url: str) -> Caption: diff --git a/nazurin/sites/misskey/config.py b/nazurin/sites/misskey/config.py index 6a2a99b4..882bb8a1 100644 --- a/nazurin/sites/misskey/config.py +++ b/nazurin/sites/misskey/config.py @@ -7,4 +7,5 @@ with env.prefixed("FILE_"): DESTINATION: str = env.str("PATH", default="Misskey") FILENAME: str = env.str( - "NAME", default="{id} - {filename} - {user[name]}({user[username]})") + "NAME", default="{id} - {filename} - {user[name]}({user[username]})" + ) From 4665bbdd33308a2fc582cd29f319c3c5207e15a9 Mon Sep 17 00:00:00 2001 From: yyoung Date: Tue, 21 Nov 2023 22:51:26 +0800 Subject: [PATCH 19/25] refactor: validate response with pydantic model --- nazurin/sites/misskey/api.py | 131 +++++++++++++------------------- nazurin/sites/misskey/config.py | 2 +- nazurin/sites/misskey/models.py | 42 ++++++++++ 3 files changed, 97 insertions(+), 78 deletions(-) create mode 100644 nazurin/sites/misskey/models.py diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index 432c9dfc..5a449888 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -1,49 +1,27 @@ import os import shlex import subprocess -from datetime import datetime from pathlib import Path from typing import List, Tuple from aiohttp.client_exceptions import ClientResponseError +from pydantic import ValidationError from nazurin.models import Caption, Illust, Image, Ugoira from nazurin.models.file import File +from nazurin.sites.misskey.models import File as NoteFile +from nazurin.sites.misskey.models import Note from nazurin.utils import Request, logger from nazurin.utils.decorators import async_wrap, network_retry from nazurin.utils.exceptions import NazurinError +from nazurin.utils.helpers import fromisoformat from .config import DESTINATION, FILENAME class Misskey: - def check_res_json(self, data: dict) -> bool: - note_required_fields = ["id", "user", "text", "createdAt", "files"] - for f in note_required_fields: - if f not in data: - return False - user = data["user"] - user_required_fileds = ["username", "name"] - for f in user_required_fileds: - if f not in user: - return False - files = data["files"] - file_required_fields = [ - "name", - "type", - "url", - "thumbnailUrl", - "size", - "properties", - ] - for file in files: - for f in file_required_fields: - if f not in file: - return False - return True - @network_retry - async def get_note(self, site_url: str, note_id: str) -> dict: + async def get_note(self, site_url: str, note_id: str) -> Note: """Fetch a note from a Misskey instance.""" api = f"https://{site_url}/api/notes/show" json = {"noteId": note_id} @@ -54,38 +32,29 @@ async def get_note(self, site_url: str, note_id: str) -> dict: response.raise_for_status() except ClientResponseError as err: raise NazurinError(err) from None - data = await response.json() - # check JSON format - if not self.check_res_json(data): - raise NazurinError("Invalid JSON format.") - - return data + data = await response.json() + try: + return Note.model_validate(data) + except ValidationError as err: + raise NazurinError(err) from None - def build_caption(self, note: dict, site_url: str) -> Caption: - url = f"https://{site_url}/notes/{note['id']}" + def build_caption(self, note: Note, site_url: str) -> Caption: + url = f"https://{site_url}/notes/{note.id}" + caption = { + "url": url, + "author": f"{note.user.name} #{note.user.username}", + "text": note.text, + } # URL from the original instance - if note["uri"] is None: - return Caption( - { - "url": url, - "author": f"{note['user']['username']} #{note['user']['name']}", - "text": note["text"], - } - ) - else: - return Caption( - { - "url": url, - "original_url": note["uri"], - "author": f"{note['user']['username']} #{note['user']['name']}", - "text": note["text"], - } - ) - - async def get_video(self, file: dict, destination: str, filename: str) -> File: - if file["type"] == "video/mp4" or file["type"] == "image/gif": - video = File(filename, file["url"], destination) + if note.uri is not None: + caption["original_url"] = note.uri + return Caption(caption) + + async def get_video(self, file: NoteFile, destination: str, filename: str) -> File: + file_type = file.type + if file_type in ["video/mp4", "image/gif"]: + video = File(filename, file.url, destination) else: @async_wrap @@ -117,7 +86,7 @@ def convert(config: File, output: File): ) raise NazurinError("Failed to convert ugoira to mp4.") from None - ori_video = File(filename, file["url"]) + ori_video = File(filename, file.url) async with Request() as session: await ori_video.download(session) filename, _ = os.path.splitext(filename) @@ -125,51 +94,59 @@ def convert(config: File, output: File): await convert(ori_video, video) return video - async def parse_note(self, note: dict, site_url: str) -> Illust: + async def parse_note(self, note: Note, site_url: str) -> Illust: """Build caption and get images.""" # Build note caption caption = self.build_caption(note, site_url) images: List[Image] = [] files: List[File] = [] - file_dict = note["files"] - for file in file_dict: - destination, filename = self.get_storage_dest(note, file["name"]) - if file["type"].startswith("image") and not file["type"].endswith("gif"): + note_files = note.files + for index, file in enumerate(note_files): + if not file.url: + continue + destination, filename = self.get_storage_dest(note, file, index) + file_type = file.type + if file_type.startswith("image") and not file_type.endswith("gif"): images.append( Image( filename, - file["url"], + file.url, destination, - file["thumbnailUrl"], - file["size"], - file["properties"]["width"], - file["properties"]["height"], + file.thumbnailUrl, + file.size, + file.properties.width, + file.properties.height, ) ) - elif file["type"].startswith("video") or file["type"].endswith("gif"): + elif file_type.startswith("video") or file_type.endswith("gif"): return Ugoira( - await self.get_video(file, destination, filename), caption, note + await self.get_video(file, destination, filename), + caption, + note.model_dump(), ) - return Illust(images, caption, note, files) + return Illust(images, caption, note.model_dump(), files) - async def fetch(self, site_url: str, post_id: str) -> Illust: - note = await self.get_note(site_url, post_id) + async def fetch(self, site_url: str, note_id: str) -> Illust: + note = await self.get_note(site_url, note_id) return await self.parse_note(note, site_url) @staticmethod - def get_storage_dest(note: dict, filename: str) -> Tuple[str, str]: + def get_storage_dest(note: Note, file: NoteFile, index: int) -> Tuple[str, str]: """ Format destination and filename. """ - # remove 'Z' to fit datetime.fromisoformat's needs - created_at = datetime.fromisoformat(note["createdAt"][:-1]) - filename, extension = os.path.splitext(filename) + created_at = fromisoformat(note.createdAt) + filename, extension = os.path.splitext(file.name) context = { - **note, + "user": note.user.model_dump(), + **file.properties.model_dump(), + "md5": file.md5, # Human-friendly filename, without extension "filename": filename, + "index": index, + "id": note.id, "created_at": created_at, "extension": extension, } diff --git a/nazurin/sites/misskey/config.py b/nazurin/sites/misskey/config.py index 882bb8a1..0daf26a6 100644 --- a/nazurin/sites/misskey/config.py +++ b/nazurin/sites/misskey/config.py @@ -7,5 +7,5 @@ with env.prefixed("FILE_"): DESTINATION: str = env.str("PATH", default="Misskey") FILENAME: str = env.str( - "NAME", default="{id} - {filename} - {user[name]}({user[username]})" + "NAME", default="{id}_{index} - {filename} - {user[name]}({user[username]})" ) diff --git a/nazurin/sites/misskey/models.py b/nazurin/sites/misskey/models.py new file mode 100644 index 00000000..56854b9d --- /dev/null +++ b/nazurin/sites/misskey/models.py @@ -0,0 +1,42 @@ +from typing import Optional + +from pydantic import BaseModel, ConfigDict + + +class User(BaseModel): + model_config = ConfigDict(extra="allow") + + id: str + username: str + name: Optional[str] + + +class FileProperties(BaseModel): + model_config = ConfigDict(extra="allow") + + width: int + height: int + + +class File(BaseModel): + model_config = ConfigDict(extra="allow") + + name: str + type: str + md5: str + size: int + properties: FileProperties + url: Optional[str] + thumbnailUrl: Optional[str] + + +class Note(BaseModel): + model_config = ConfigDict(extra="allow") + + id: str + createdAt: str + userId: str + user: User + text: Optional[str] + files: list[File] + uri: Optional[str] = None From 9c0764845bf96cdfda1376d62754d60b26529396 Mon Sep 17 00:00:00 2001 From: yyoung Date: Tue, 21 Nov 2023 22:57:24 +0800 Subject: [PATCH 20/25] refactor: return Misskey error message --- nazurin/sites/misskey/api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index 5a449888..0d18861b 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -28,6 +28,12 @@ async def get_note(self, site_url: str, note_id: str) -> Note: async with Request() as request: async with request.post(url=api, json=json) as response: + if response.status == 400: + result = await response.json() + error = result["error"] + raise NazurinError( + f"Error: {error['message']} ({error['code']})" + ) from None try: response.raise_for_status() except ClientResponseError as err: From 9f6efd82f39a0185761259942970855c18ea985b Mon Sep 17 00:00:00 2001 From: yyoung Date: Tue, 21 Nov 2023 23:00:16 +0800 Subject: [PATCH 21/25] fix: requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 4044bfdf..e7c4b735 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ environs~=9.3.4 async_lru~=2.0.2 loguru~=0.6.0 humanize~=4.8.0 +pydantic~=2.5.1 pixivpy3~=3.7.2 beautifulsoup4~=4.10.0 From 82d4433bf8dd416472a76dbfb1cf6ae4ad7b9c2a Mon Sep 17 00:00:00 2001 From: yyoung Date: Tue, 21 Nov 2023 23:04:24 +0800 Subject: [PATCH 22/25] refactor: update match pattern for note ID --- nazurin/sites/misskey/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nazurin/sites/misskey/interface.py b/nazurin/sites/misskey/interface.py index 5abd74a9..00a766ed 100644 --- a/nazurin/sites/misskey/interface.py +++ b/nazurin/sites/misskey/interface.py @@ -8,7 +8,7 @@ patterns = [ # https://site.example/notes/9khcu788zb - r"://(.*?)/notes/(.*)", + r"https?://(.*?)/notes/([0-9a-z]+)", ] From 71bb37127cebb296ed409b0847d61606315d14bf Mon Sep 17 00:00:00 2001 From: yyoung Date: Wed, 22 Nov 2023 22:34:17 +0800 Subject: [PATCH 23/25] fix: Python 3.8 typing syntax --- nazurin/sites/misskey/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nazurin/sites/misskey/models.py b/nazurin/sites/misskey/models.py index 56854b9d..ee7370d2 100644 --- a/nazurin/sites/misskey/models.py +++ b/nazurin/sites/misskey/models.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import List, Optional from pydantic import BaseModel, ConfigDict @@ -38,5 +38,5 @@ class Note(BaseModel): userId: str user: User text: Optional[str] - files: list[File] + files: List[File] uri: Optional[str] = None From 5ed8e0b61c9518a38dbf84fe5760554a94d824b0 Mon Sep 17 00:00:00 2001 From: Kare-Udon Date: Sat, 2 Dec 2023 02:35:28 +0000 Subject: [PATCH 24/25] Fix: note visibility check in Misskey API --- nazurin/sites/misskey/api.py | 15 ++++++++------- nazurin/sites/misskey/models.py | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index 0d18861b..7cb755b8 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -41,7 +41,9 @@ async def get_note(self, site_url: str, note_id: str) -> Note: data = await response.json() try: - return Note.model_validate(data) + note = Note.model_validate(data) + if note.visibility not in ["public", "home"]: + raise NazurinError("Note is not public.") except ValidationError as err: raise NazurinError(err) from None @@ -59,10 +61,7 @@ def build_caption(self, note: Note, site_url: str) -> Caption: async def get_video(self, file: NoteFile, destination: str, filename: str) -> File: file_type = file.type - if file_type in ["video/mp4", "image/gif"]: - video = File(filename, file.url, destination) - else: - + if file_type not in ["video/mp4", "image/gif"]: @async_wrap def convert(config: File, output: File): config_path = Path(config.path).as_posix() @@ -90,14 +89,16 @@ def convert(config: File, output: File): error.returncode, error.output.decode(), ) - raise NazurinError("Failed to convert ugoira to mp4.") from None - + raise NazurinError( + "Failed to convert ugoira to mp4.") from None ori_video = File(filename, file.url) async with Request() as session: await ori_video.download(session) filename, _ = os.path.splitext(filename) video = File(filename + ".mp4", "", destination) await convert(ori_video, video) + else: + video = File(filename, file.url, destination) return video async def parse_note(self, note: Note, site_url: str) -> Illust: diff --git a/nazurin/sites/misskey/models.py b/nazurin/sites/misskey/models.py index ee7370d2..490eef55 100644 --- a/nazurin/sites/misskey/models.py +++ b/nazurin/sites/misskey/models.py @@ -40,3 +40,4 @@ class Note(BaseModel): text: Optional[str] files: List[File] uri: Optional[str] = None + visibility: str From 64a190ffbf595b1ba8bd7f5d919d1d03bf91da9a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 2 Dec 2023 02:35:57 +0000 Subject: [PATCH 25/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nazurin/sites/misskey/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nazurin/sites/misskey/api.py b/nazurin/sites/misskey/api.py index 7cb755b8..ce46f1fe 100644 --- a/nazurin/sites/misskey/api.py +++ b/nazurin/sites/misskey/api.py @@ -62,6 +62,7 @@ def build_caption(self, note: Note, site_url: str) -> Caption: async def get_video(self, file: NoteFile, destination: str, filename: str) -> File: file_type = file.type if file_type not in ["video/mp4", "image/gif"]: + @async_wrap def convert(config: File, output: File): config_path = Path(config.path).as_posix() @@ -89,8 +90,8 @@ def convert(config: File, output: File): error.returncode, error.output.decode(), ) - raise NazurinError( - "Failed to convert ugoira to mp4.") from None + raise NazurinError("Failed to convert ugoira to mp4.") from None + ori_video = File(filename, file.url) async with Request() as session: await ori_video.download(session)