From 7c25c7b4a9f214f7e8cd768c97e76012d17ddc6a Mon Sep 17 00:00:00 2001 From: Maxim Slipenko Date: Mon, 6 Jan 2025 11:40:39 +0300 Subject: [PATCH] fix: add tmp chat_member handler --- src/gnomik/poetry.lock | 15 ++++ .../karkas_blocks/standard/spam/main.py | 24 +++++- .../karkas_blocks/standard/statistics/main.py | 15 +++- src/karkas_blocks/poetry.lock | 18 ++++- src/karkas_blocks/pyproject.toml | 1 + src/karkas_core/karkas_core/logger.py | 77 ++++++++++++++----- src/karkas_core/karkas_core/main.py | 13 +++- .../modules_system/public_api/__init__.py | 1 + .../modules_system/public_api/public_api.py | 5 ++ src/karkas_core/karkas_core/singleton.py | 1 + src/karkas_core/poetry.lock | 15 ++++ src/karkas_lite/poetry.lock | 17 +++- 12 files changed, 173 insertions(+), 29 deletions(-) diff --git a/src/gnomik/poetry.lock b/src/gnomik/poetry.lock index b89f3ad..15addd6 100644 --- a/src/gnomik/poetry.lock +++ b/src/gnomik/poetry.lock @@ -1190,6 +1190,7 @@ dash-bootstrap-components = "^1.6.0" dash-extensions = "^1.0.18" karkas-core = {path = "../karkas_core", develop = true} peewee = "^3.17.6" +python-json-logger = "^3.2.1" pyyaml = "^6.0.1" torch = "^2.0" transformers = "^4.45.2" @@ -1858,6 +1859,20 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "python-json-logger" +version = "3.2.1" +description = "JSON Log Formatter for the Python Logging Package" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python_json_logger-3.2.1-py3-none-any.whl", hash = "sha256:cdc17047eb5374bd311e748b42f99d71223f3b0e186f4206cc5d52aefe85b090"}, + {file = "python_json_logger-3.2.1.tar.gz", hash = "sha256:8eb0554ea17cb75b05d2848bc14fb02fbdbd9d6972120781b974380bfa162008"}, +] + +[package.extras] +dev = ["backports.zoneinfo", "black", "build", "freezegun", "mdx_truly_sane_lists", "mike", "mkdocs", "mkdocs-awesome-pages-plugin", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-material (>=8.5)", "mkdocstrings[python]", "msgspec", "msgspec-python313-pre", "mypy", "orjson", "pylint", "pytest", "tzdata", "validate-pyproject[all]"] + [[package]] name = "python-multipart" version = "0.0.9" diff --git a/src/karkas_blocks/karkas_blocks/standard/spam/main.py b/src/karkas_blocks/karkas_blocks/standard/spam/main.py index ab0901c..7120ed6 100644 --- a/src/karkas_blocks/karkas_blocks/standard/spam/main.py +++ b/src/karkas_blocks/karkas_blocks/standard/spam/main.py @@ -148,7 +148,14 @@ class SpamDetectionMiddleware(BaseMiddleware): if not verified_users_row["verified"]: if event.text: is_spam, confidence = spam_predict(event.text) - log(f"{event.text} is {is_spam} with confidence {confidence}") + + log( + "message spam confidence", + trace_id=data["trace_id"], + is_spam=is_spam, + confidence=confidence, + ) + if is_spam: if confidence >= config.get( "spam::spam_detection::confidence" @@ -165,6 +172,11 @@ class SpamDetectionMiddleware(BaseMiddleware): == f"{event.chat.id}-{event.from_user.id}", ) + log( + "user is not spammer anymore", + trace_id=data["trace_id"], + ) + result = await handler(event, data) return result @@ -180,8 +192,10 @@ class ChatMemberMiddleware(BaseMiddleware): event: "ChatMemberUpdated", data: Dict[str, Any], ) -> Any: + log("chat member middleware - start") if await joinFilter(event): + log("chat member join") try: await VerifiedUsers.insert( VerifiedUsers( @@ -194,6 +208,8 @@ class ChatMemberMiddleware(BaseMiddleware): except Exception as e: log(str(e)) + log("chat member middleware - ok") + result = await handler(event, data) return result @@ -215,6 +231,7 @@ def module_init(): register_outer_chat_member_middleware(ChatMemberMiddleware()) register_app_config = get_module("standard.database", "register_app_config") + SimpleAdminFilter: "Type[ISimpleAdminFilter]" = get_module( "standard.filters", "SimpleAdminFilter" ) @@ -225,6 +242,11 @@ def module_init(): router = Router() + def tmp(event): + pass + + router.chat_member.register(tmp) + router.message.register( delete_spam_by_request, SimpleAdminFilter(), Command("spam") ) diff --git a/src/karkas_blocks/karkas_blocks/standard/statistics/main.py b/src/karkas_blocks/karkas_blocks/standard/statistics/main.py index 374df56..afd3ba7 100644 --- a/src/karkas_blocks/karkas_blocks/standard/statistics/main.py +++ b/src/karkas_blocks/karkas_blocks/standard/statistics/main.py @@ -1,3 +1,4 @@ +import uuid from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict from aiogram import BaseMiddleware @@ -9,7 +10,7 @@ if TYPE_CHECKING: from karkas_blocks.standard.config import IConfig -from karkas_core.modules_system.public_api import get_module +from karkas_core.modules_system.public_api import get_module, log config: "IConfig" = get_module("standard.config", "config") @@ -69,9 +70,19 @@ class StatisticsMiddleware(BaseMiddleware): async def __call__( self, handler: Callable[["TelegramObject", Dict[str, Any]], Awaitable[Any]], - event: "TelegramObject", + event: "Message", data: Dict[str, Any], ) -> Any: + trace_id = str(uuid.uuid4()) + + log( + "new message recieved", + trace_id=trace_id, + chat_id=event.chat.id, + message_text=event.text, + ) + + data["trace_id"] = trace_id await update_chat_stats(event) await update_user_stats(event) diff --git a/src/karkas_blocks/poetry.lock b/src/karkas_blocks/poetry.lock index 0b13731..1d19b8e 100644 --- a/src/karkas_blocks/poetry.lock +++ b/src/karkas_blocks/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiofiles" @@ -1438,6 +1438,20 @@ files = [ [package.dependencies] typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" +[[package]] +name = "python-json-logger" +version = "3.2.1" +description = "JSON Log Formatter for the Python Logging Package" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python_json_logger-3.2.1-py3-none-any.whl", hash = "sha256:cdc17047eb5374bd311e748b42f99d71223f3b0e186f4206cc5d52aefe85b090"}, + {file = "python_json_logger-3.2.1.tar.gz", hash = "sha256:8eb0554ea17cb75b05d2848bc14fb02fbdbd9d6972120781b974380bfa162008"}, +] + +[package.extras] +dev = ["backports.zoneinfo", "black", "build", "freezegun", "mdx_truly_sane_lists", "mike", "mkdocs", "mkdocs-awesome-pages-plugin", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-material (>=8.5)", "mkdocstrings[python]", "msgspec", "msgspec-python313-pre", "mypy", "orjson", "pylint", "pytest", "tzdata", "validate-pyproject[all]"] + [[package]] name = "pyyaml" version = "6.0.2" @@ -2311,4 +2325,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "e4c8f543df7eea0b8410cb3ef189099ce1ea14082051fa40c96ce7f4ce3e0513" +content-hash = "a1fd839181bf4b4f84df4ec18be43b4a78e2b0292a3d421bd0974ae3bd4f49fa" diff --git a/src/karkas_blocks/pyproject.toml b/src/karkas_blocks/pyproject.toml index dab8d5a..5dbcd9f 100644 --- a/src/karkas_blocks/pyproject.toml +++ b/src/karkas_blocks/pyproject.toml @@ -19,6 +19,7 @@ dash-bootstrap-components = "^1.6.0" torch = { version = "^2.0", source="pytorch-cpu" } transformers = "^4.45.2" +python-json-logger = "^3.2.1" [tool.poetry-monorepo.deps] enabled = true diff --git a/src/karkas_core/karkas_core/logger.py b/src/karkas_core/karkas_core/logger.py index 620de07..c163e55 100644 --- a/src/karkas_core/karkas_core/logger.py +++ b/src/karkas_core/karkas_core/logger.py @@ -1,36 +1,73 @@ import logging +import sys import traceback +from io import TextIOWrapper -app_logger = logging.getLogger("karkas") -log_level = logging.INFO +from pythonjsonlogger import jsonlogger + +# Константы для настройки логгера +DEFAULT_LOG_LEVEL = logging.INFO +LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s %(message)s" +DATE_FORMAT = "%Y-%m-%d %H:%M:%S" -def patch_logger(logger_: logging.Logger): - logger_.handlers = [] - formatter = logging.Formatter("%(asctime)s %(message)s", datefmt="%H:%M:%S") - console_handler = logging.StreamHandler() +def configure_logger(logger: logging.Logger, level=DEFAULT_LOG_LEVEL) -> logging.Logger: + """ + Конфигурация логгера с использованием JSON-форматтера. + """ + logger.handlers = [] # Удалить существующие обработчики + logger.propagate = False # Отключить проброс логов + logger.setLevel(level) + + formatter = jsonlogger.JsonFormatter( + fmt=LOG_FORMAT, datefmt=DATE_FORMAT, json_ensure_ascii=False + ) + + console_handler = logging.StreamHandler( + stream=TextIOWrapper(sys.stdout.buffer, encoding="utf-8") + ) console_handler.setFormatter(formatter) - logger_.addHandler(console_handler) - logger_.propagate = False - logger_.setLevel(log_level) - return logger_ + + logger.addHandler(console_handler) + return logger -def setup_logger(): +def get_logger(name: str, level=DEFAULT_LOG_LEVEL) -> logging.Logger: """ - Настройка логирования + Возвращает настроенный логгер с заданным именем. """ - patch_logger(app_logger) + logger = logging.getLogger(name) + return configure_logger(logger, level) -def log(message): +# Глобальный логгер приложения +app_logger = get_logger("karkas") + + +def log(message, level="info", **kwargs): + """ + Логирование сообщений с поддержкой уровней. + Уровни: debug, info, warning, error, critical. + Поддерживает логирование исключений и дополнительных полей. + """ + # Проверка на исключение if isinstance(message, Exception): - error_message = f"Error: {str(message)}\n{traceback.format_exc()}" - app_logger.error(error_message) - else: - app_logger.info(message) + message = {"error": str(message), "traceback": traceback.format_exc()} + + # Выбор метода логирования + log_method = getattr(app_logger, level, app_logger.info) + + # Добавление пользовательских данных + if kwargs: + if isinstance(message, dict): + message.update(kwargs) + else: + message = {"message": message, **kwargs} + + log_method(message) +# Пример интеграции с Hypercorn try: from hypercorn.logging import Logger as HypercornLogger @@ -38,9 +75,9 @@ try: def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) if self.error_logger: - patch_logger(self.error_logger) + configure_logger(self.error_logger) if self.access_logger: - patch_logger(self.access_logger) + configure_logger(self.access_logger) except ImportError: pass diff --git a/src/karkas_core/karkas_core/main.py b/src/karkas_core/karkas_core/main.py index 2a68060..e195913 100644 --- a/src/karkas_core/karkas_core/main.py +++ b/src/karkas_core/karkas_core/main.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING from aiogram import Bot, Dispatcher -from karkas_core.logger import log, setup_logger +from karkas_core.logger import log from karkas_core.modules_system import ModulesManager from karkas_core.modules_system.public_api import get_module from karkas_core.singleton import Singleton @@ -17,7 +17,6 @@ class Karkas: pass async def init_app(self, bot_modules, metainfo=None): - setup_logger() singleton = Singleton() if isinstance(metainfo, dict): singleton.storage["metainfo"] = metainfo @@ -41,9 +40,15 @@ class Karkas: singleton.dp.include_routers(*singleton.storage["_routers"]) for middleware in singleton.storage["_outer_message_middlewares"]: + log("update outer middleware register", middleware=middleware) singleton.dp.message.outer_middleware.register(middleware) + for middleware in singleton.storage["_outer_update_middlewares"]: + log("message outer middleware register", middleware=middleware) + singleton.dp.update.outer_middleware.register(middleware) + for middleware in singleton.storage["_outer_chat_member_middlewares"]: + log("chat_member outer middleware register", middleware=middleware) singleton.dp.chat_member.outer_middleware.register(middleware) await singleton.modules_manager.late_init() @@ -65,7 +70,9 @@ class Karkas: async def start_long_polling_mode(self): singleton = Singleton() await singleton.bot.delete_webhook() - await singleton.dp.start_polling(singleton.bot) + await singleton.dp.start_polling( + singleton.bot, + ) async def start_webhook_mode(self): try: diff --git a/src/karkas_core/karkas_core/modules_system/public_api/__init__.py b/src/karkas_core/karkas_core/modules_system/public_api/__init__.py index b567ded..3a9016a 100644 --- a/src/karkas_core/karkas_core/modules_system/public_api/__init__.py +++ b/src/karkas_core/karkas_core/modules_system/public_api/__init__.py @@ -7,6 +7,7 @@ from .public_api import ( get_module, register_outer_chat_member_middleware, register_outer_message_middleware, + register_outer_update_middleware, register_router, set_chat_menu_button, set_my_commands, diff --git a/src/karkas_core/karkas_core/modules_system/public_api/public_api.py b/src/karkas_core/karkas_core/modules_system/public_api/public_api.py index 4c21d8f..1b7128c 100644 --- a/src/karkas_core/karkas_core/modules_system/public_api/public_api.py +++ b/src/karkas_core/karkas_core/modules_system/public_api/public_api.py @@ -28,6 +28,11 @@ def register_outer_message_middleware(middleware: "BaseMiddleware"): app.storage["_outer_message_middlewares"].append(middleware) +def register_outer_update_middleware(middleware: "BaseMiddleware"): + app = Singleton() + app.storage["_outer_update_middlewares"].append(middleware) + + def register_outer_chat_member_middleware(middleware: "BaseMiddleware"): app = Singleton() app.storage["_outer_chat_member_middlewares"].append(middleware) diff --git a/src/karkas_core/karkas_core/singleton.py b/src/karkas_core/karkas_core/singleton.py index 80db906..3723d3e 100644 --- a/src/karkas_core/karkas_core/singleton.py +++ b/src/karkas_core/karkas_core/singleton.py @@ -26,6 +26,7 @@ class Singleton(metaclass=SingletonMeta): "_fsm_storage": MemoryStorage(), "_routers": [], "_outer_message_middlewares": [], + "_outer_update_middlewares": [], "_outer_chat_member_middlewares": [], "metainfo": {}, } diff --git a/src/karkas_core/poetry.lock b/src/karkas_core/poetry.lock index fa5929f..dba1fe3 100644 --- a/src/karkas_core/poetry.lock +++ b/src/karkas_core/poetry.lock @@ -1076,6 +1076,7 @@ dash-bootstrap-components = "^1.6.0" dash-extensions = "^1.0.18" karkas-core = {path = "../karkas_core", develop = true} peewee = "^3.17.6" +python-json-logger = "^3.2.1" pyyaml = "^6.0.1" torch = "^2.0" transformers = "^4.45.2" @@ -1645,6 +1646,20 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "python-json-logger" +version = "3.2.1" +description = "JSON Log Formatter for the Python Logging Package" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python_json_logger-3.2.1-py3-none-any.whl", hash = "sha256:cdc17047eb5374bd311e748b42f99d71223f3b0e186f4206cc5d52aefe85b090"}, + {file = "python_json_logger-3.2.1.tar.gz", hash = "sha256:8eb0554ea17cb75b05d2848bc14fb02fbdbd9d6972120781b974380bfa162008"}, +] + +[package.extras] +dev = ["backports.zoneinfo", "black", "build", "freezegun", "mdx_truly_sane_lists", "mike", "mkdocs", "mkdocs-awesome-pages-plugin", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-material (>=8.5)", "mkdocstrings[python]", "msgspec", "msgspec-python313-pre", "mypy", "orjson", "pylint", "pytest", "tzdata", "validate-pyproject[all]"] + [[package]] name = "python-multipart" version = "0.0.9" diff --git a/src/karkas_lite/poetry.lock b/src/karkas_lite/poetry.lock index 53358eb..99c0450 100644 --- a/src/karkas_lite/poetry.lock +++ b/src/karkas_lite/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiofiles" @@ -1198,6 +1198,7 @@ dash-bootstrap-components = "^1.6.0" dash-extensions = "^1.0.18" karkas-core = {path = "../karkas_core", develop = true} peewee = "^3.17.6" +python-json-logger = "^3.2.1" pyyaml = "^6.0.1" torch = "^2.0" transformers = "^4.45.2" @@ -1980,6 +1981,20 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "python-json-logger" +version = "3.2.1" +description = "JSON Log Formatter for the Python Logging Package" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python_json_logger-3.2.1-py3-none-any.whl", hash = "sha256:cdc17047eb5374bd311e748b42f99d71223f3b0e186f4206cc5d52aefe85b090"}, + {file = "python_json_logger-3.2.1.tar.gz", hash = "sha256:8eb0554ea17cb75b05d2848bc14fb02fbdbd9d6972120781b974380bfa162008"}, +] + +[package.extras] +dev = ["backports.zoneinfo", "black", "build", "freezegun", "mdx_truly_sane_lists", "mike", "mkdocs", "mkdocs-awesome-pages-plugin", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-material (>=8.5)", "mkdocstrings[python]", "msgspec", "msgspec-python313-pre", "mypy", "orjson", "pylint", "pytest", "tzdata", "validate-pyproject[all]"] + [[package]] name = "python-multipart" version = "0.0.12"