Merged with feat/add-spam-block

.gitignore

@@ -8,3 +8,4 @@ __pycache__
 Karkas.db
 config.yaml
 dist
+*.py[cod]

gitflic-ci.yaml

@@ -46,3 +46,12 @@ build-gnomik:
     IMAGE_NAME: registry.gitflic.ru/project/alt-gnome/karkas/gnomik
     KARKAS_PROJECT: gnomik
   extends: .docker-dev-build-template
+
+build-karkas-lite:
+  stage: build
+  image: docker:27.1.2
+  variables:
+    CI_REGISTRY: registry.gitflic.ru
+    IMAGE_NAME: registry.gitflic.ru/project/alt-gnome/karkas/karkas-lite
+    KARKAS_PROJECT: karkas_lite
+  extends: .docker-dev-build-template

karkas.code-workspace

@@ -23,6 +23,10 @@
 		{
 			"name": "ALT Linux",
 			"path": "src/altlinux"
+		},
+		{
+			"name": "Karkas Lite",
+			"path": "src/karkas_lite"
 		}
 	],
 	"extensions": {

src/gnomik/gnomik/__main__.py

@@ -20,7 +20,6 @@ async def main():
             block_loader("external", "create_report_apps"),
             block_loader("standard", "info"),
             block_loader("standard", "help"),
-
             # block_loader("external", "yandexgpt", safe=False),
             #
             # block_loader("standard", "admin"),

src/karkas_blocks/karkas_blocks/standard/admin/handlers.py

@@ -43,20 +43,20 @@ async def chat_not_in_approve_list(message: Message, bot: Bot):
 async def mute_user(chat_id: int, user_id: int, time: int, bot: Bot):
     # TODO: type variable using `typing`
     mutePermissions = {
-        "can_send_messages": False,         # bool | None
-        "can_send_audios": False,           # bool | None
-        "can_send_documents": False,        # bool | None
-        "can_send_photos": False,           # bool | None
-        "can_send_videos": False,           # bool | None
-        "can_send_video_notes": False,      # bool | None
-        "can_send_voice_notes": False,      # bool | None
-        "can_send_polls": False,            # bool | None
-        "can_send_other_messages": False,   # bool | None
-        "can_add_web_page_previews": False, # bool | None
-        "can_change_info": False,           # bool | None
-        "can_invite_users": False,          # bool | None
-        "can_pin_messages": False,          # bool | None
-        "can_manage_topics": False,         # bool | None
+        "can_send_messages": False,  # bool | None
+        "can_send_audios": False,  # bool | None
+        "can_send_documents": False,  # bool | None
+        "can_send_photos": False,  # bool | None
+        "can_send_videos": False,  # bool | None
+        "can_send_video_notes": False,  # bool | None
+        "can_send_voice_notes": False,  # bool | None
+        "can_send_polls": False,  # bool | None
+        "can_send_other_messages": False,  # bool | None
+        "can_add_web_page_previews": False,  # bool | None
+        "can_change_info": False,  # bool | None
+        "can_invite_users": False,  # bool | None
+        "can_pin_messages": False,  # bool | None
+        "can_manage_topics": False,  # bool | None
         # **extra_data: Any
     }
     end_time = time + int(time.time())

src/karkas_blocks/karkas_blocks/standard/command_helper/main.py

@@ -5,18 +5,22 @@ from karkas_core.modules_system.public_api import set_my_commands
 commands = dict()
 
 
-def register_command(command, description, role="USER"):
+def register_command(command, description, long_description=None, role="USER"):
+    if long_description is None:
+        long_description = description
+
     if role not in commands:
         commands[role] = dict()
     commands[role][command] = {
         "description": description,
+        "long_description": long_description,
     }
 
 
-async def set_user_commands():
+async def set_commands(role="USER"):
     bot_commands = []
-    if "USER" in commands:
-        user_commands = commands["USER"]
+    if role in commands:
+        user_commands = commands[role]
         for command in user_commands:
             bot_commands.append(
                 BotCommand(
@@ -25,18 +29,24 @@ async def set_user_commands():
                 )
             )
 
-    # log(bot_commands)
-
     await set_my_commands(
         bot_commands,
     )
 
 
-def get_user_commands():
-    if "USER" in commands:
-        return commands["USER"].copy()
+def get_commands(role="USER"):
+    if role in commands:
+        return commands[role].copy()
     return {}
 
 
+async def set_user_commands():
+    await set_commands("USER")
+
+
+def get_user_commands():
+    return get_commands("USER")
+
+
 async def module_late_init():
     await set_user_commands()

src/karkas_blocks/karkas_blocks/standard/filters/__init__.py

@@ -2,6 +2,7 @@ from .filters import (
     ChatIDFilter,
     ChatModerOrAdminFilter,
     ChatNotInApproveFilter,
+    SimpleAdminFilter,
     chat_not_in_approve,
     module_init,
 )

src/karkas_blocks/karkas_blocks/standard/filters/filters.py

@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict
 from aiogram import BaseMiddleware, Bot
 from aiogram.filters import BaseFilter
 from aiogram.types import Message, TelegramObject
+from aiogram.utils.chat_member import ADMINS
 from typing_extensions import deprecated
 
 from karkas_core.modules_system.public_api import (
@@ -121,3 +122,9 @@ class ChatModerOrAdminFilter(BaseFilter):
             or await roles.check_moderator_permission(user_id)
             or any(user_id == admin.user.id for admin in admins)
         )
+
+
+class SimpleAdminFilter(BaseFilter):
+    async def __call__(self, message: Message, bot: Bot) -> bool:
+        member = await bot.get_chat_member(message.chat.id, message.from_user.id)
+        return isinstance(member, ADMINS)

src/karkas_blocks/karkas_blocks/standard/help/main.py

@@ -42,7 +42,7 @@ FOOTER = """===============
 def format_commands(commands_dict):
     formatted_commands = []
     for command, details in commands_dict.items():
-        formatted_commands.append(f"/{command} - {details['description']}")
+        formatted_commands.append(f"/{command} - {details['long_description']}")
     return "\n".join(formatted_commands)
 
 

src/karkas_blocks/karkas_blocks/standard/miniapp/lib.py

@@ -55,9 +55,7 @@ def create_dash_app(requests_pathname_prefix: str = None) -> dash.Dash:
             dbc.themes.BOOTSTRAP,
             dbc.icons.BOOTSTRAP,
         ],
-        external_scripts=[
-            "https://telegram.org/js/telegram-web-app.js"
-        ],
+        external_scripts=["https://telegram.org/js/telegram-web-app.js"],
         server=server,
         requests_pathname_prefix=real_prefix,
         routes_pathname_prefix="/_internal/",

src/karkas_blocks/karkas_blocks/standard/spam/__init__.py

@@ -0,0 +1 @@
+from .main import module_init

src/karkas_blocks/karkas_blocks/standard/spam/db/__init__.py

@@ -0,0 +1 @@
+from .piccolo_app import APP_CONFIG

src/karkas_blocks/karkas_blocks/standard/spam/db/piccolo_app.py

@@ -0,0 +1,15 @@
+import os
+
+from karkas_piccolo.conf.apps import AppConfig
+
+from .tables import SpamLog
+
+CURRENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
+
+APP_CONFIG = AppConfig(
+    app_name="standard.spam",
+    migrations_folder_path=os.path.join(CURRENT_DIRECTORY, "piccolo_migrations"),
+    table_classes=[SpamLog],
+    migration_dependencies=[],
+    commands=[],
+)

src/karkas_blocks/karkas_blocks/standard/spam/db/piccolo_migrations/standardspam_2024_10_03t17_43_50_559465.py

@@ -0,0 +1,125 @@
+from piccolo.apps.migrations.auto.migration_manager import MigrationManager
+from piccolo.columns.column_types import JSON, Date, Integer, Text
+from piccolo.columns.defaults.date import DateNow
+from piccolo.columns.indexes import IndexMethod
+
+ID = "2024-10-03T17:43:50:559465"
+VERSION = "1.16.0"
+DESCRIPTION = ""
+
+
+async def forwards():
+    manager = MigrationManager(
+        migration_id=ID, app_name="standard.spam", description=DESCRIPTION
+    )
+
+    manager.add_table(
+        class_name="SpamLog", tablename="spam_log", schema=None, columns=None
+    )
+
+    manager.add_column(
+        table_class_name="SpamLog",
+        tablename="spam_log",
+        column_name="chat_id",
+        db_column_name="chat_id",
+        column_class_name="Integer",
+        column_class=Integer,
+        params={
+            "default": 0,
+            "null": False,
+            "primary_key": False,
+            "unique": False,
+            "index": False,
+            "index_method": IndexMethod.btree,
+            "choices": None,
+            "db_column_name": None,
+            "secret": False,
+        },
+        schema=None,
+    )
+
+    manager.add_column(
+        table_class_name="SpamLog",
+        tablename="spam_log",
+        column_name="user_id",
+        db_column_name="user_id",
+        column_class_name="Integer",
+        column_class=Integer,
+        params={
+            "default": 0,
+            "null": False,
+            "primary_key": False,
+            "unique": False,
+            "index": False,
+            "index_method": IndexMethod.btree,
+            "choices": None,
+            "db_column_name": None,
+            "secret": False,
+        },
+        schema=None,
+    )
+
+    manager.add_column(
+        table_class_name="SpamLog",
+        tablename="spam_log",
+        column_name="message_text",
+        db_column_name="message_text",
+        column_class_name="Text",
+        column_class=Text,
+        params={
+            "default": "",
+            "null": True,
+            "primary_key": False,
+            "unique": False,
+            "index": False,
+            "index_method": IndexMethod.btree,
+            "choices": None,
+            "db_column_name": None,
+            "secret": False,
+        },
+        schema=None,
+    )
+
+    manager.add_column(
+        table_class_name="SpamLog",
+        tablename="spam_log",
+        column_name="attachments",
+        db_column_name="attachments",
+        column_class_name="JSON",
+        column_class=JSON,
+        params={
+            "default": "{}",
+            "null": False,
+            "primary_key": False,
+            "unique": False,
+            "index": False,
+            "index_method": IndexMethod.btree,
+            "choices": None,
+            "db_column_name": None,
+            "secret": False,
+        },
+        schema=None,
+    )
+
+    manager.add_column(
+        table_class_name="SpamLog",
+        tablename="spam_log",
+        column_name="created_at",
+        db_column_name="created_at",
+        column_class_name="Date",
+        column_class=Date,
+        params={
+            "default": DateNow(),
+            "null": False,
+            "primary_key": False,
+            "unique": False,
+            "index": False,
+            "index_method": IndexMethod.btree,
+            "choices": None,
+            "db_column_name": None,
+            "secret": False,
+        },
+        schema=None,
+    )
+
+    return manager

src/karkas_blocks/karkas_blocks/standard/spam/db/piccolo_migrations/standardspam_2024_10_03t19_24_33_522177.py

@@ -0,0 +1,29 @@
+from piccolo.apps.migrations.auto.migration_manager import MigrationManager
+
+ID = "2024-10-03T19:24:33:522177"
+VERSION = "1.16.0"
+DESCRIPTION = ""
+
+
+async def forwards():
+    manager = MigrationManager(
+        migration_id=ID, app_name="standard.spam", description=DESCRIPTION
+    )
+
+    manager.drop_column(
+        table_class_name="SpamLog",
+        tablename="spam_log",
+        column_name="chat_id",
+        db_column_name="chat_id",
+        schema=None,
+    )
+
+    manager.drop_column(
+        table_class_name="SpamLog",
+        tablename="spam_log",
+        column_name="user_id",
+        db_column_name="user_id",
+        schema=None,
+    )
+
+    return manager

src/karkas_blocks/karkas_blocks/standard/spam/db/tables.py

@@ -0,0 +1,8 @@
+from piccolo.columns import JSON, Date, Text
+from piccolo.table import Table
+
+
+class SpamLog(Table):
+    message_text = Text(null=True)
+    attachments = JSON()
+    created_at = Date()

src/karkas_blocks/karkas_blocks/standard/spam/info.json

@@ -0,0 +1,17 @@
+{
+    "id": "standard.spam",
+    "name": "Spam",
+    "description": "Модуль для удаления спама",
+    "author": "Karkas Team",
+    "version": "1.0.0",
+    "privileged": true,
+    "dependencies": {
+      "required": {
+        "standard.config": "^1.0.0"
+      },
+      "optional": {
+        "standard.command_helper": "^1.0.0",
+        "standard.filters": "^1.0.0"
+      }
+    }
+  }

src/karkas_blocks/karkas_blocks/standard/spam/main.py

@@ -0,0 +1,89 @@
+from typing import TYPE_CHECKING, Type
+
+from aiogram import Bot, Router
+from aiogram.filters import Command
+
+from karkas_core.modules_system.public_api import get_module, register_router
+
+from .db.tables import SpamLog
+
+try:
+    register_command = get_module("standard.command_helper", "register_command")
+    COMMAND_HELPER_MODULE_LOADED = True
+except Exception:
+    COMMAND_HELPER_MODULE_LOADED = False
+    pass
+
+if TYPE_CHECKING:
+    from aiogram.types import Message
+
+    from karkas_blocks.standard.filters import SimpleAdminFilter as ISimpleAdminFilter
+
+
+async def spam(message: "Message", bot: "Bot"):
+    if not message.reply_to_message:
+        return
+
+    if message.reply_to_message.from_user.id in (message.from_user.id, bot.id):
+        return
+
+    spam_message = message.reply_to_message
+    chat_id = message.chat.id
+
+    message.reply_to_message.media_group_id
+
+    attachments = {
+        "version": 1,
+        "photo": (
+            [size.model_dump() for size in spam_message.photo]
+            if spam_message.photo
+            else []
+        ),
+        "gif": spam_message.animation.model_dump() if spam_message.animation else [],
+    }
+
+    spam_log = SpamLog(
+        message_text=spam_message.text,
+        attachments=attachments,
+    )
+
+    await bot.delete_message(
+        chat_id=chat_id,
+        message_id=spam_message.message_id,
+    )
+
+    await bot.delete_message(
+        chat_id=chat_id,
+        message_id=message.message_id,
+    )
+
+    await bot.ban_chat_member(
+        chat_id=chat_id,
+        user_id=spam_message.from_user.id,
+    )
+
+    await SpamLog.insert(spam_log)
+
+
+def module_init():
+    register_app_config = get_module("standard.database", "register_app_config")
+    SimpleAdminFilter: "Type[ISimpleAdminFilter]" = get_module(
+        "standard.filters", "SimpleAdminFilter"
+    )
+    from .db import APP_CONFIG
+
+    register_app_config(APP_CONFIG)
+
+    router = Router()
+
+    router.message.register(spam, SimpleAdminFilter(), Command("spam"))
+
+    register_router(router)
+
+    if COMMAND_HELPER_MODULE_LOADED:
+        register_command(
+            "spam",
+            "Удалить спам и забанить пользователя",
+            long_description="Удалить спам и забанить пользователя. "
+            "Собирает обезличенные данные для дальнейшего создания алгоритма",
+        )

src/karkas_blocks/karkas_blocks/standard/statistics/db/tables.py

@@ -10,7 +10,8 @@ class ChatStats(Table):
 
 class Messages(Table):
     # Key format: `{message_chat_id}-{message_id}`
-    # (A temporary measure until https://github.com/piccolo-orm/piccolo/pull/984 is accepted)
+    # (A temporary measure until piccolo-orm/piccolo#984 is accepted)
+    #
     key = Text(primary_key=True)
 
     chat_id = Integer()
@@ -23,7 +24,7 @@ class Messages(Table):
 
 class UserStats(Table):
     # Key format: `{chat_id}-{user_id}`
-    # (A temporary measure until https://github.com/piccolo-orm/piccolo/pull/984 is accepted)
+    # (A temporary measure until piccolo-orm/piccolo#984 is accepted)
     key = Text(primary_key=True)
 
     chat_id = Integer()

src/karkas_blocks/karkas_blocks/standard/welcome/main.py

@@ -100,7 +100,7 @@ async def left_member_handler(event: "ChatMemberUpdated", bot: Bot):
     task = verification_tasks.get(key)
     await task.end(success=False)
 
-    verification_tasks.pop((user_id, chat_id), None)
+    verification_tasks.remove(key_from_user_chat(user_id, chat_id))
 
 
 async def verify_timeout(task: BaseTask):

src/karkas_lite/Dockerfile

@@ -0,0 +1,38 @@
+FROM python:3.12-slim AS dependencies_installer
+
+RUN pip install poetry
+
+WORKDIR /app
+
+COPY ./src/karkas_core/poetry* ./src/karkas_core/pyproject.toml  /app/src/karkas_core/
+COPY ./src/karkas_blocks/poetry* ./src/karkas_blocks/pyproject.toml /app/src/karkas_blocks/
+COPY ./src/karkas_piccolo/poetry* ./src/karkas_piccolo/pyproject.toml /app/src/karkas_piccolo/
+COPY ./src/karkas_lite/poetry* ./src/karkas_lite/pyproject.toml /app/src/karkas_lite/
+
+WORKDIR /app/src/karkas_lite
+
+RUN poetry install --no-root --no-directory
+
+FROM python:3.12-slim AS src
+
+COPY ./src/karkas_core /app/src/karkas_core
+COPY ./src/karkas_blocks /app/src/karkas_blocks
+COPY ./src/karkas_piccolo /app/src/karkas_piccolo
+COPY ./src/karkas_lite /app/src/karkas_lite
+
+FROM python:3.12-slim AS local_dependencies_installer
+
+RUN pip install poetry
+
+COPY --from=dependencies_installer /app/src/karkas_lite/.venv /app/src/karkas_lite/.venv
+COPY --from=src /app/src/ /app/src/
+WORKDIR /app/src/karkas_lite
+RUN poetry install
+
+FROM python:3.12-slim AS base
+COPY --from=local_dependencies_installer /app/src/karkas_lite/.venv /app/src/karkas_lite/.venv
+COPY --from=src /app/src/ /app/src/
+WORKDIR /app/src/karkas_lite
+ENV PATH="/app/.venv/bin:$PATH"
+
+CMD ["/bin/bash", "-c", ". .venv/bin/activate && python -m karkas_lite"]

src/karkas_lite/Dockerfile.dockerignore

@@ -0,0 +1,14 @@
+**/Dockerfile
+**/*.dockerignore
+**/docker-compose.yml
+
+**/.git
+**/.gitignore
+
+**/.venv
+
+**/.mypy_cache
+**/__pycache__/
+
+src/karkas_lite/config.yaml
+src/karkas_lite/database/*

src/karkas_lite/config-example.yaml

@@ -0,0 +1,3 @@
+core:
+  mode: LONG_POLLING
+  token: xxx

src/karkas_lite/docker-compose.yml

@@ -0,0 +1,10 @@
+services:
+  app:
+    build:
+      context: ../..
+      dockerfile: src/karkas_lite/Dockerfile
+    ports:
+      - 9000:9000
+    volumes:
+      - ./config.yaml:/app/src/karkas_lite/config.yaml
+      - ./database:/app/database

src/karkas_lite/karkas_lite/__main__.py

@@ -0,0 +1,24 @@
+import asyncio
+
+from karkas_blocks import block_loader
+from karkas_core import Karkas
+
+
+async def main():
+    ocab = Karkas()
+    await ocab.init_app(
+        [
+            block_loader("standard", "config", safe=False),
+            block_loader("standard", "filters", safe=False),
+            block_loader("standard", "database", safe=False),
+            block_loader("standard", "command_helper"),
+            block_loader("standard", "spam", safe=False),
+            block_loader("standard", "report"),
+            block_loader("standard", "welcome", safe=False),
+            block_loader("standard", "help"),
+        ]
+    )
+    await ocab.start()
+
+
+asyncio.run(main())

src/karkas_lite/poetry.toml

@@ -0,0 +1,2 @@
+[virtualenvs]
+in-project = true

src/karkas_lite/pyproject.toml

@@ -0,0 +1,18 @@
+[tool.poetry]
+name = "karkas-lite"
+version = "0.1.0"
+description = ""
+authors = [
+    "Maxim Slipenko <maxim@slipenko.com>"
+]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.13"
+karkas-core = { extras=["webhook"], path = "../karkas_core", develop = true }
+karkas-blocks = { path = "../karkas_blocks", develop = true }
+karkas-piccolo = { path = "../karkas_piccolo", develop = true }
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"