feat: добавил подписчиков на удаленные директории с выгрузкой новых файлов

data/connection.json

@@ -0,0 +1,6 @@
+{
+  "host": "172.21.5.240",
+  "port": 22,
+  "username": "smart",
+  "password": "00000000"
+}

data/path.json

@@ -0,0 +1,14 @@
+[
+  {
+    "source": "/home/smart/PN/TWC/trace/Weld Point",
+    "buffer": "/home/andrei/PycharmProjects/traceDelivery/buffer",
+    "destination": "/home/andrei/Desktop/bla",
+    "apply_filter": false
+  },
+  {
+    "source": "/home/smart/PN/TWC/trace/test1",
+    "buffer": "/home/andrei/PycharmProjects/traceDelivery/buffer",
+    "destination": "/home/andrei/Desktop/bla",
+    "apply_filter": false
+  }
+]

requirements.txt

@@ -0,0 +1,13 @@
+bcrypt==4.2.0
+cffi==1.17.1
+cryptography==43.0.3
+loguru==0.7.2
+numpy==2.1.3
+pandas==2.2.3
+paramiko==3.5.0
+pycparser==2.22
+PyNaCl==1.5.0
+python-dateutil==2.9.0.post0
+pytz==2024.2
+six==1.16.0
+tzdata==2024.2

src/main.py

@@ -0,0 +1,50 @@
+import json
+import os
+from concurrent.futures import ThreadPoolExecutor
+
+from remote.client import SSHClient
+from remote.connection_data import SSHConnectionParams
+from subscription.subscription_object import SubscriptionObject
+from subscription.subscriber import Subscriber
+from tuning.tuner import Tuner
+
+
+def init_files_buffer(subscription_objects) -> None:
+    for element in subscription_objects:
+        if not os.path.exists(element.buffer):
+            os.makedirs(element.buffer)
+
+
+def read_json(filepath: str) -> dict:
+    if not os.path.exists(filepath):
+        raise FileNotFoundError(f"File {filepath} not found!")
+
+    with open(filepath, 'r') as json_file:
+        content = json.load(json_file)
+        return content
+
+
+def start_subscription(ssh_connection_object: SSHConnectionParams,
+          subscription_objects: list[SubscriptionObject]) -> None:
+    tuner = Tuner()
+    executor = ThreadPoolExecutor()
+
+    for element in subscription_objects:
+        client = SSHClient(ssh_connection_object)
+        client.connect()
+        subscriber = Subscriber(client.sftp, element)
+        tuner.accept(subscriber)
+        executor.submit(subscriber.subscribe)
+
+
+def main():
+    ssh_connection_dict = read_json("../data/connection.json")
+    ssh_connection_object = SSHConnectionParams(**ssh_connection_dict)
+    subscription_objects_list = read_json("../data/path.json")
+    subscription_objects = [SubscriptionObject(**element) for element in subscription_objects_list]
+    init_files_buffer(subscription_objects)
+    start_subscription(ssh_connection_object, subscription_objects)
+
+
+if __name__ == '__main__':
+    main()

src/remote/client.py

@@ -0,0 +1,36 @@
+from typing import Optional
+
+import paramiko
+
+from remote.connection_data import SSHConnectionParams
+
+
+class SSHClient:
+    def __init__(self, connection_params: SSHConnectionParams):
+        self._connection_params = connection_params
+
+        self._transport: Optional[paramiko.Transport] = None
+        self._sftp: Optional[paramiko.SFTPClient] = None
+
+    @property
+    def connection_params(self) -> SSHConnectionParams:
+        return self._connection_params
+
+    @property
+    def sftp(self) -> Optional[paramiko.SFTPClient]:
+        return self._sftp
+
+    @property
+    def transport(self) -> Optional[paramiko.Transport]:
+        return self._transport
+
+    def connect(self) -> None:
+        self._transport = paramiko.Transport((self._connection_params.host,
+                                              self._connection_params.port))
+        self._transport.connect(username=self._connection_params.username,
+                                password=self._connection_params.password)
+        self._sftp = paramiko.SFTPClient.from_transport(self._transport)
+
+    def disconnect(self) -> None:
+        self._sftp.close()
+        self._transport.close()

src/remote/connection_data.py

@@ -0,0 +1,8 @@
+from typing import NamedTuple
+
+
+class SSHConnectionParams(NamedTuple):
+    host: str
+    port: int
+    username: str
+    password: str

src/subscription/base.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+import paramiko
+
+from subscription.subscription_object import SubscriptionObject
+from tuning.tuner import Tuner
+
+
+class BaseSubscriber:
+
+    def __init__(self,
+                 sftp_client: paramiko.SFTPClient,
+                 subscription_object: SubscriptionObject,
+                 tuner: Optional[Tuner] = None):
+        self._sftp_client = sftp_client
+        self._subscription_object = subscription_object
+        self._tuner = tuner
+
+        self._history = []
+
+    @property
+    def sftp_client(self) -> paramiko.SFTPClient:
+        return self._sftp_client
+
+    @property
+    def subscription_object(self) -> SubscriptionObject:
+        return self._subscription_object
+
+    @property
+    def tuner(self) -> Optional[Tuner]:
+        return self._tuner
+
+    @tuner.setter
+    def tuner(self, tuner: Tuner) -> None:
+        self._tuner = tuner

src/subscription/subscriber.py

@@ -0,0 +1,34 @@
+from os import path
+from time import sleep
+
+from loguru import logger
+
+from subscription.base import BaseSubscriber
+
+
+class Subscriber(BaseSubscriber):
+
+    def _init_state(self):
+        files = self._sftp_client.listdir(self._subscription_object.source)
+        self._history = files
+        logger.info(f"Subscription to {self._subscription_object.source} done!")
+
+    def _download_file(self, file: str):
+        remote_path = path.join(self._subscription_object.source, file)
+        local_path = path.join(self._subscription_object.buffer, file)
+        self._sftp_client.get(remotepath=remote_path, localpath=local_path)
+        logger.info(f"Download: {remote_path} --> {local_path}")
+        self._tuner.notify(self._subscription_object, file)
+
+    def subscribe(self):
+        self._init_state()
+
+        while True:
+            files = self._sftp_client.listdir(self._subscription_object.source)
+            new_files = list(filter(lambda x: x not in self._history, files))
+            if new_files:
+                logger.info(f"New files: {new_files} in {self._subscription_object.source} found!")
+                self._history = files
+                for file in new_files:
+                    self._download_file(file)
+            sleep(0.01)

src/subscription/subscription_object.py

@@ -0,0 +1,8 @@
+from typing import NamedTuple
+
+
+class SubscriptionObject(NamedTuple):
+    source: str
+    buffer: str
+    destination: str
+    apply_filter: bool

src/tuning/abstraction.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+
+from subscription.subscription_object import SubscriptionObject
+
+
+class AbstractTuner(ABC):
+
+    busy: bool = False
+
+    @abstractmethod
+    def accept(self, subscriber) -> None:
+        ...
+
+    @abstractmethod
+    def notify(self, obj: SubscriptionObject, filename: str):
+        ...

src/tuning/tools.py

@@ -0,0 +1,11 @@
+
+def filter_signal(signal_name: str):
+    ...
+
+
+def move_signal(signal_name: str):
+    ...
+
+
+def convert_bool_to_int(signal_name: str):
+    ...

src/tuning/tuner.py

@@ -0,0 +1,21 @@
+from os import path
+import shutil
+
+from tuning.abstraction import AbstractTuner
+from subscription.subscription_object import SubscriptionObject
+import tuning.tools
+
+
+class Tuner(AbstractTuner):
+
+    def accept(self, subscriber) -> None:
+        subscriber.tuner = self
+
+    def notify(self, obj: SubscriptionObject, filename: str):
+        local_path = path.join(obj.buffer, filename)
+        destination_path = path.join(obj.destination, filename)
+
+        if obj.apply_filter:
+            ...
+        else:
+            shutil.move(src=local_path, dst=destination_path)