From 3a35c77419658ec0b09715f6f48994e0ebe03021 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Tue, 8 Jul 2025 01:13:58 +0400 Subject: [PATCH 1/9] fix --- .deploy/deploy-dev.yaml | 23 ++++++++++- .deploy/deploy-prod.yaml | 21 ++++++++++ helpers/configurator.py | 84 +++++++++++++++++++++++++++++++++++++++ helpers/queues.py | 85 ++++++++++++++++++++++++++++++++++++++++ main.py | 36 +++++++++++++++++ 5 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 helpers/configurator.py create mode 100644 helpers/queues.py diff --git a/.deploy/deploy-dev.yaml b/.deploy/deploy-dev.yaml index a9abe02..efcb71b 100644 --- a/.deploy/deploy-dev.yaml +++ b/.deploy/deploy-dev.yaml @@ -46,8 +46,29 @@ services: parallelism: 1 order: start-first + bot: + image: mathwave/sprint-repo:b-jokes + environment: + MONGO_HOST: "mongo.develop.sprinthub.ru" + MONGO_PASSWORD: $MONGO_PASSWORD_DEV + command: bot + networks: + - configurator + - queues-development + deploy: + mode: replicated + restart_policy: + condition: any + update_config: + parallelism: 1 + order: start-first + networks: b-jokes-net: driver: overlay common-infra-nginx: - external: true \ No newline at end of file + external: true + configurator: + external: true + queues-development: + external: true diff --git a/.deploy/deploy-prod.yaml b/.deploy/deploy-prod.yaml index 7dd5055..60305b2 100644 --- a/.deploy/deploy-prod.yaml +++ b/.deploy/deploy-prod.yaml @@ -60,8 +60,29 @@ services: parallelism: 1 order: start-first + bot: + image: mathwave/sprint-repo:b-jokes + environment: + MONGO_HOST: "mongo.sprinthub.ru" + MONGO_PASSWORD: $MONGO_PASSWORD_PROD + command: bot + networks: + - configurator + - queues + deploy: + mode: replicated + restart_policy: + condition: any + update_config: + parallelism: 1 + order: start-first + networks: b-jokes-net: driver: overlay common-infra-nginx: external: true + configurator: + external: true + queues: + external: true diff --git a/helpers/configurator.py b/helpers/configurator.py new file mode 100644 index 0000000..54dd496 --- /dev/null +++ b/helpers/configurator.py @@ -0,0 +1,84 @@ +import json +import urllib.parse +from threading import Thread +from time import sleep + +from requests import get + + +class ConfiguratorClient: + def __init__(self, app_name: str, stage: str, need_poll: bool = True): + self.app_name = app_name + self.stage = stage + self.endpoint = 'http://configurator/' + self.fetch_url = urllib.parse.urljoin(self.endpoint, '/api/v1/fetch') + self.config_storage = {} + self.experiment_storage = {} + self.staff_storage = {} + self.poll_data() + if need_poll: + self.poll_data_in_thread() + + def poll_data_in_thread(self): + def inner(): + while True: + sleep(30) + self.fetch() + + Thread(target=inner, daemon=True).start() + + def poll_data(self): + self.fetch(with_exception=True) + + def request_with_retries(self, url, params, with_exception=False, retries_count=3): + exception_to_throw = None + for _ in range(retries_count): + try: + response = get( + url, + params=params + ) + if response.status_code == 200: + return response.json() + print(f'Failed to request {url}, status_code={response.status_code}') + exception_to_throw = Exception('Not 200 status') + except Exception as exc: + print(exc) + exception_to_throw = exc + sleep(1) + print(f'Failed fetching with retries: {url}, {params}') + if with_exception: + raise exception_to_throw + + def fetch(self, with_exception=False): + if self.stage == 'local': + local_platform = json.loads(open('local_platform.json', 'r').read()) + self.config_storage = local_platform['configs'] + self.experiment_storage = local_platform['experiments'] + self.staff_storage = { + key: set(value) + for key, value in local_platform['platform_staff'].items() + } + return + response_data = self.request_with_retries(self.fetch_url, { + 'project': self.app_name, + 'stage': self.stage, + }, with_exception) + self.config_storage = response_data['configs'] + self.experiment_storage = response_data['experiments'] + self.staff_storage = { + key: set(value) + for key, value in response_data['platform_staff'].items() + } + + def is_staff(self, **kwargs): + for key, value in kwargs.items(): + if value in self.staff_storage[key]: + return True + return False + + def get_config(self, name): + return self.config_storage[name] + + def get_experiment(self, name): + return self.experiment_storage[name] diff --git a/helpers/queues.py b/helpers/queues.py new file mode 100644 index 0000000..e8054a7 --- /dev/null +++ b/helpers/queues.py @@ -0,0 +1,85 @@ +from concurrent.futures import ThreadPoolExecutor +import datetime +import os +import zoneinfo +import requests +import time + + +stage = os.getenv("STAGE", 'local') +if stage == 'local': + QUEUES_URL = 'http://localhost:1239' +else: + QUEUES_URL = 'http://queues:1239' + + +class QueuesException(Exception): + ... + + +class TasksHandlerMixin: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.executor = ThreadPoolExecutor(max_workers=1) + + def _send_metric(self, start: datetime.datetime, end: datetime.datetime, success: bool): + def send(): + requests.post(f'{QUEUES_URL}/api/v1/metric', json={ + 'service': 'b-jokes', + 'queue': self.queue_name, + 'success': success, + 'timestamp': start.strftime("%Y-%m-%dT%H:%M:%S") + "Z", + "success": success, + "execution_time_ms": (end - start).microseconds // 1000, + "environment": stage, + }) + + self.executor.submit(send) + + def poll(self): + while True: + try: + response = requests.get(f'{QUEUES_URL}/api/v1/take', headers={'queue': self.queue_name}).json() + except requests.JSONDecodeError: + print('Unable to decode json') + time.sleep(3) + continue + task = response.get('task') + if not task: + time.sleep(0.2) + continue + start = datetime.datetime.now(zoneinfo.ZoneInfo("Europe/Moscow")) + try: + print(f'process task with id {task["id"]}, attempt {task["attempt"]}') + self.process(task['payload']) + success = True + except Exception as exc: + print(f'Error processing message id={task["id"]}, payload={task["payload"]}, exc={exc}') + success = False + end = datetime.datetime.now(zoneinfo.ZoneInfo("Europe/Moscow")) + if success: + try: + resp = requests.post(f'{QUEUES_URL}/api/v1/finish', json={'id': task['id']}) + if resp.status_code != 202: + raise QueuesException + print(f'finish task with id {task["id"]}') + except: + print(f'Failed to finish task id={task["id"]}') + self._send_metric(start, end, success) + + @property + def queue_name(self): + raise NotImplemented + + def process(self, payload): + raise NotImplemented + + +def set_task(queue_name: str, payload: dict, seconds_to_execute: int, delay: int|None = None): + resp = requests.post(f'{QUEUES_URL}/api/v1/put', headers={'queue': queue_name}, json={ + 'payload': payload, + 'seconds_to_execute': seconds_to_execute, + 'delay': delay, + }) + if resp.status_code != 202: + raise QueuesException diff --git a/main.py b/main.py index e481dfa..4f46461 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,9 @@ +import os from flask import Flask, request, make_response +from helpers.configurator import ConfiguratorClient +from helpers.jokes import get_random +from helpers.queues import TasksHandlerMixin, set_task import settings from helpers.events import events from processor import Processor @@ -38,3 +42,35 @@ def run(): a = 1 / 0 app.run(host="0.0.0.0", port=8000, debug=settings.DEBUG) + + +def bot(): + configurator = ConfiguratorClient("b-jokes", os.getenv("STAGE", "local")) + class Bot(TasksHandlerMixin): + @property + def queue_name(self): + return "b_jokes_worker" + + def process(self, payload): + text = payload.get('text') + if not text: + return + for word in configurator.get_config('words'): + if word in text: + mes = 'Держи шутку!\n' + get_random() + set_task( + "botalka_mailbox", + { + 'project': 'b-jokes', + 'name': 'telegram-bot', + 'body': { + 'text': mes, + 'reply_to_message_id': payload['message_id'], + 'chat_id': payload['chat']['id'], + } + }, + 1 + ) + return + + Bot().poll() From 556a45b0d46554ca68caf587737560a45b649142 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Tue, 8 Jul 2025 01:19:01 +0400 Subject: [PATCH 2/9] fix --- .gitea/workflows/deploy-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/deploy-dev.yaml b/.gitea/workflows/deploy-dev.yaml index 9b429e1..a1c7cae 100644 --- a/.gitea/workflows/deploy-dev.yaml +++ b/.gitea/workflows/deploy-dev.yaml @@ -44,4 +44,4 @@ jobs: - name: deploy env: MONGO_PASSWORD_DEV: ${{ secrets.MONGO_PASSWORD_DEV }} - run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-dev.yaml b-jokes + run: docker stack deploy --with-registry-auth -c ./.deploy/deploy-dev.yaml b-jokes-development From 10f86f2f99015f2b5eb50d69d1ec1ad734676dd9 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Tue, 8 Jul 2025 01:21:29 +0400 Subject: [PATCH 3/9] fix --- .gitea/workflows/deploy-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/deploy-dev.yaml b/.gitea/workflows/deploy-dev.yaml index a1c7cae..341ec45 100644 --- a/.gitea/workflows/deploy-dev.yaml +++ b/.gitea/workflows/deploy-dev.yaml @@ -32,7 +32,7 @@ jobs: run: docker push mathwave/sprint-repo:b-jokes-nginx deploy-dev: name: Deploy dev - runs-on: [dev] + runs-on: [prod] needs: push steps: - name: login From c15e950e027a5780579e82de21f2788a7a64eb26 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Tue, 8 Jul 2025 01:25:34 +0400 Subject: [PATCH 4/9] fix --- entrypoint.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/entrypoint.py b/entrypoint.py index d3e0e89..d7f70d0 100644 --- a/entrypoint.py +++ b/entrypoint.py @@ -12,5 +12,7 @@ if arg == "poll": jokes.poll_jokes() elif arg == "api": main.run() +elif arg == "bot": + main.bot() else: raise NotImplementedError("No arg specified!") From 28df4aef548d539b2d499dfc8a8d1f31fb37cec1 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Tue, 8 Jul 2025 01:30:10 +0400 Subject: [PATCH 5/9] fix --- .deploy/deploy-dev.yaml | 1 + .deploy/deploy-prod.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.deploy/deploy-dev.yaml b/.deploy/deploy-dev.yaml index efcb71b..e3a1357 100644 --- a/.deploy/deploy-dev.yaml +++ b/.deploy/deploy-dev.yaml @@ -51,6 +51,7 @@ services: environment: MONGO_HOST: "mongo.develop.sprinthub.ru" MONGO_PASSWORD: $MONGO_PASSWORD_DEV + STAGE: "development" command: bot networks: - configurator diff --git a/.deploy/deploy-prod.yaml b/.deploy/deploy-prod.yaml index d88f483..9d3eaeb 100644 --- a/.deploy/deploy-prod.yaml +++ b/.deploy/deploy-prod.yaml @@ -53,6 +53,7 @@ services: environment: MONGO_HOST: "mongo.sprinthub.ru" MONGO_PASSWORD: $MONGO_PASSWORD_PROD + STAGE: "production" command: bot networks: - configurator From 6509e58b6edec934ff5694f6e4a9b4954609d311 Mon Sep 17 00:00:00 2001 From: Egor Matveev Date: Tue, 8 Jul 2025 02:05:25 +0400 Subject: [PATCH 6/9] fix --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 4f46461..baabc71 100644 --- a/main.py +++ b/main.py @@ -55,6 +55,7 @@ def bot(): text = payload.get('text') if not text: return + text = text.lower() for word in configurator.get_config('words'): if word in text: mes = 'Держи шутку!\n' + get_random() From 6a1f2d3d09ac691b2a5456671877f1e913c6d902 Mon Sep 17 00:00:00 2001 From: emmatveev Date: Fri, 12 Sep 2025 20:42:09 +0300 Subject: [PATCH 7/9] Update .deploy/deploy-dev.yaml --- .deploy/deploy-dev.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.deploy/deploy-dev.yaml b/.deploy/deploy-dev.yaml index e3a1357..c602fdb 100644 --- a/.deploy/deploy-dev.yaml +++ b/.deploy/deploy-dev.yaml @@ -21,7 +21,7 @@ services: networks: - b-jokes-net environment: - MONGO_HOST: "mongo.develop.sprinthub.ru" + MONGO_HOST: "mongo.dev.chocomarsh.com" MONGO_PASSWORD: $MONGO_PASSWORD_DEV command: api deploy: @@ -35,7 +35,7 @@ services: poll: image: mathwave/sprint-repo:b-jokes environment: - MONGO_HOST: "mongo.develop.sprinthub.ru" + MONGO_HOST: "mongo.dev.chocomarsh.com" MONGO_PASSWORD: $MONGO_PASSWORD_DEV command: poll deploy: @@ -49,7 +49,7 @@ services: bot: image: mathwave/sprint-repo:b-jokes environment: - MONGO_HOST: "mongo.develop.sprinthub.ru" + MONGO_HOST: "mongo.dev.chocomarsh.com" MONGO_PASSWORD: $MONGO_PASSWORD_DEV STAGE: "development" command: bot From 8cdf0cbd8dfdd0cbb5198f3e4b4b114c74f10c5f Mon Sep 17 00:00:00 2001 From: emmatveev Date: Fri, 12 Sep 2025 20:42:31 +0300 Subject: [PATCH 8/9] Update .deploy/deploy-prod.yaml --- .deploy/deploy-prod.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.deploy/deploy-prod.yaml b/.deploy/deploy-prod.yaml index 9d3eaeb..7205cfc 100644 --- a/.deploy/deploy-prod.yaml +++ b/.deploy/deploy-prod.yaml @@ -21,7 +21,7 @@ services: networks: - b-jokes-net environment: - MONGO_HOST: "mongo.sprinthub.ru" + MONGO_HOST: "mongo.chocomarsh.com" MONGO_PASSWORD: $MONGO_PASSWORD_PROD DEBUG: "false" command: api @@ -36,7 +36,7 @@ services: poll: image: mathwave/sprint-repo:b-jokes environment: - MONGO_HOST: "mongo.sprinthub.ru" + MONGO_HOST: "mongo.chocomarsh.com" MONGO_PASSWORD: $MONGO_PASSWORD_PROD DEBUG: "false" command: poll @@ -51,7 +51,7 @@ services: bot: image: mathwave/sprint-repo:b-jokes environment: - MONGO_HOST: "mongo.sprinthub.ru" + MONGO_HOST: "mongo.chocomarsh.com" MONGO_PASSWORD: $MONGO_PASSWORD_PROD STAGE: "production" command: bot From f165ee6f30d9535732cc921b3bc98db2335384d8 Mon Sep 17 00:00:00 2001 From: emmatveev Date: Sun, 14 Sep 2025 09:45:04 +0300 Subject: [PATCH 9/9] Update .deploy/deploy-prod.yaml --- .deploy/deploy-prod.yaml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.deploy/deploy-prod.yaml b/.deploy/deploy-prod.yaml index 7205cfc..2abc212 100644 --- a/.deploy/deploy-prod.yaml +++ b/.deploy/deploy-prod.yaml @@ -48,30 +48,8 @@ services: parallelism: 1 order: start-first - bot: - image: mathwave/sprint-repo:b-jokes - environment: - MONGO_HOST: "mongo.chocomarsh.com" - MONGO_PASSWORD: $MONGO_PASSWORD_PROD - STAGE: "production" - command: bot - networks: - - configurator - - queues - deploy: - mode: replicated - restart_policy: - condition: any - update_config: - parallelism: 1 - order: start-first - networks: b-jokes-net: driver: overlay common-infra-nginx: external: true - configurator: - external: true - queues: - external: true