add mask_db_delete.py, amp_mask_db_writer.py,

add amp_mask_template_creator.py

masks/amp_mask_db_writer.py

@@ -0,0 +1,38 @@
+import datetime
+import time
+
+import pandas as pd
+
+# import numpy as np
+import pymongo
+
+from config_info.config import DB_URL
+
+# from file_reader.db_file_reader import DbFileReader
+
+db_client = pymongo.MongoClient(DB_URL)
+prisma_db = db_client["prisma-32_db"]
+
+
+def amp_mask_db_writer(cluster, start_date, end_date, path_to_mask):
+    """Запись из excel файла данных амплитудной маски в БД, за указанный промежуток времени."""
+    # Кажется нужно указать только год и месяц, который мы хотим записать в БД из excel.
+    mask_amp = pd.read_excel(f'{path_to_mask}\\{cluster}cl_amp_mask_{start_date.year}.xlsx',
+                             sheet_name=f'{start_date.year}-{start_date.month:02}')
+    for i in mask_amp.index:
+        daily_mask_data = mask_amp[[f"amp{i}_mask" for i in range(1, 17)]].iloc[0, :].tolist()
+        collection_prisma = prisma_db[f'{str(mask_amp["date"][i].date())}_12d']
+        upd_result = collection_prisma.update_many({'cluster': cluster},
+                                                   {"$set": {
+                                                       f'mask_of_hit_counters_amp': int(
+                                                           ''.join(map(lambda x: str(x), daily_mask_data)), 2),
+                                                       f'multiplicity_of_hit_counters_amp': sum(daily_mask_data)}})
+        print(f'Matched documents amp_mask_{cluster} - {upd_result.matched_count}')
+        print(f'Modified documents amp_mask_{cluster} - {upd_result.modified_count}')
+
+
+if __name__ == "__main__":
+    t1 = time.time_ns()
+    amp_mask_db_writer(cluster=1, start_date=datetime.date(2021, 1, 1), end_date=datetime.date(2021, 1, 31),
+                       path_to_mask="C:\\Users\\pad_z\\OneDrive\\Рабочий стол\\PrismaPassport\\amp_mask")
+    print(f'time - {(time.time_ns() - t1) / 1e9}')

masks/amp_mask_template_creator.py

@@ -0,0 +1,28 @@
+import datetime
+import time
+
+import pandas as pd
+import numpy as np
+
+
+def amp_mask_template_creator(cluster, year, path_to_mask):
+    """Создание excel файла за год, в котором прописывается шаблон амплитудной маски."""
+    with pd.ExcelWriter(f'{path_to_mask}\\{cluster}cl_amp_mask_{year}.xlsx') as writer:
+        for period in pd.period_range(start=datetime.date(year, 1, 1), end=datetime.date(year, 12, 31), freq='M'):
+            start_month = datetime.date(period.year, period.month, 1)
+            end_month = datetime.date(period.year + int(period.month / 12), ((period.month % 12) + 1),
+                                      1) - datetime.timedelta(days=1)
+            daterange = [single_date.date() for single_date in pd.date_range(start_month, end_month)]
+            amp_template = np.ones((len(daterange), 16))
+            mask_amp = pd.DataFrame(amp_template.astype(int),
+                                    columns=[f'amp{i}_mask' for i in range(1, 17)])
+            mask_amp['date'] = daterange
+            mask_amp = mask_amp[['date'] + [f'amp{i}_mask' for i in range(1, 17)]]
+            mask_amp.to_excel(writer, sheet_name=str(period), index=False)
+
+
+if __name__ == "__main__":
+    t1 = time.time_ns()
+    amp_mask_template_creator(cluster=1, year=2022,
+                              path_to_mask="C:\\Users\\pad_z\\OneDrive\\Рабочий стол\\PrismaPassport\\amp_mask")
+    print(f'time - {(time.time_ns() - t1) / 1e9}')

masks/mask_db_delete.py

@@ -0,0 +1,35 @@
+import datetime
+import time
+
+import pandas as pd
+
+# import numpy as np
+import pymongo
+
+from config_info.config import DB_URL
+
+# from file_reader.db_file_reader import DbFileReader
+
+db_client = pymongo.MongoClient(DB_URL)
+prisma_db = db_client["prisma-32_db"]
+
+
+def mask_db_delete(cluster, start_date, end_date, mask_type):
+    """Удаление данных нейтронной или амплитудной маски, за выделенный промежуток времени."""
+    for single_date in pd.date_range(start_date, end_date):
+        collection_prisma = prisma_db[f'{str(single_date.date())}_12d']
+        upd_result = collection_prisma.update_many({'cluster': cluster},
+                                                   {"$unset": {
+                                                       f'mask_of_hit_counters_{mask_type}': "",
+                                                       f'multiplicity_of_hit_counters_{mask_type}': ""}})
+        print(f'Matched documents {mask_type}_mask_{cluster} - {upd_result.matched_count}')
+        print(f'Modified documents {mask_type}_mask_{cluster} - {upd_result.modified_count}')
+
+
+if __name__ == "__main__":
+    t1 = time.time_ns()
+    mask_db_delete(cluster=1, start_date=datetime.date(2021, 1, 1), end_date=datetime.date(2021, 1, 31),
+                   mask_type='amp')
+    mask_db_delete(cluster=2, start_date=datetime.date(2021, 1, 1), end_date=datetime.date(2021, 1, 31),
+                   mask_type='amp')
+    print(f'time - {(time.time_ns() - t1) / 1e9}')

masks/neutron_mask_db_writer.py

@@ -9,11 +9,13 @@ from file_reader.db_file_reader import DbFileReader
 
 
 def neutron_mask_db_writer(cluster, start_date, end_date, path_to_db):
+    """Запись нейтронной маски в БД по критерию >0.1 за день."""
     for single_date in pd.date_range(start_date, end_date):
         db_file_reader = DbFileReader(cluster=cluster, single_date=single_date, db_url=path_to_db)
         data_cl = db_file_reader.reading_db()
         neutron_to_zero_trigger = ProccessingPrismaCl._neutron_to_zero_trigger(n_file=data_cl)
-        n_mask_params = [int(val) for val in list(map(lambda x: x < 0.1, neutron_to_zero_trigger))]
+        n_mask_params = [int(val) for val in list(map(lambda x: x < 0.1, neutron_to_zero_trigger))]  # Посмотреть
+        # в каком порядке парсится маска нейтронов, то есть от 1-го до 16-го, или от 16-го до 1-го.
         daily_binary_n_mask = {
             'mask_of_hit_counters_n': int(''.join([str(val) for val in n_mask_params]), 2),
             'multiplicity_of_hit_counters_n': sum(n_mask_params)}
@@ -29,4 +31,4 @@ if __name__ == "__main__":
                            path_to_db="mongodb://localhost:27017/")
     neutron_mask_db_writer(cluster=2, start_date=datetime.date(2018, 2, 1), end_date=datetime.date(2018, 3, 31),
                            path_to_db="mongodb://localhost:27017/")
-    print(f'time - {(time.time_ns()-t1)/1e9}')
+    print(f'time - {(time.time_ns() - t1) / 1e9}')

masks/sqlmask_converter.py

@@ -0,0 +1,45 @@
+import datetime
+
+import sqlalchemy
+import pandas as pd
+import pymongo
+
+from config_info.config import DB_URL
+
+db_client = pymongo.MongoClient(DB_URL)
+prisma_db = db_client["prisma-32_db"]
+
+
+def sqlmask_converter_to_nosql(cluster, start_date, end_date, mask_type):
+    """Переводим маску из SQL БД в бинарный вид для транспортировки в noSQL БД.
+    SQL вид существует для 2012-02-01::2021-12-31"""
+    conn = 'postgresql+psycopg2://postgres:qwerty@localhost:5432/prisma'
+    engine = sqlalchemy.create_engine(conn)
+    connect = engine.connect()
+    mask_prisma = pd.read_sql(
+        f"SELECT * FROM mask_{cluster}_params WHERE date >= "
+        f"'{start_date.year}-{start_date.month:02}-{start_date.day:02}' AND  date <= \
+           '{end_date.year}-{end_date.month:02}-{end_date.day:02}' ORDER BY date asc;", connect)
+    data_prisma_mask = mask_prisma[[f'{mask_type}{i}_mask' for i in range(16, 0, -1)]]
+
+    for i in data_prisma_mask.index:
+        mask_params = data_prisma_mask.iloc[i]
+
+        collection_prisma = prisma_db[f'{str(mask_prisma["date"][i])}_12d']
+        upd_result = collection_prisma.update_many({'cluster': cluster},
+                                                   {"$set": {
+                                                       f'mask_of_hit_counters_{mask_type}': int(
+                                                           ''.join(mask_params.astype(str)), 2),
+                                                       f'multiplicity_of_hit_counters_{mask_type}': sum(mask_params)}})
+        print(f'Matched documents {mask_type}_mask_{cluster} - {upd_result.matched_count}')
+        print(f'Modified documents {mask_type}_mask_{cluster} - {upd_result.modified_count}')
+
+
+if __name__ == '__main__':
+    date_time_start = datetime.date(2021, 12, 1)
+    date_time_stop = datetime.date(2021, 12, 31)
+    sqlmask_converter_to_nosql(cluster=1, start_date=date_time_start, end_date=date_time_stop, mask_type='amp')
+    sqlmask_converter_to_nosql(cluster=2, start_date=date_time_start, end_date=date_time_stop, mask_type='amp')
+    sqlmask_converter_to_nosql(cluster=1, start_date=date_time_start, end_date=date_time_stop, mask_type='n')
+    sqlmask_converter_to_nosql(cluster=2, start_date=date_time_start, end_date=date_time_stop, mask_type='n')
+    print('test')

sqlmask_converter.py

@@ -1,73 +0,0 @@
-import pandas as pd
-import datetime
-import sqlalchemy
-
-
-def sqlmask_converter(start_date, end_date):
-    """Переводим маску из SQL БД в бинарный вид для транспортировки в noSQL БД"""
-    conn = 'postgresql+psycopg2://postgres:qwerty@localhost:5432/prisma'
-    engine = sqlalchemy.create_engine(conn)
-    connect = engine.connect()
-    mask_prisma_1 = pd.read_sql("SELECT * FROM mask_1_params WHERE date >= '{}-{:02}-{:02}' AND  date <= \
-           '{}-{:02}-{:02}' ORDER BY date asc;".format(start_date.year, start_date.month, start_date.day, end_date.year,
-                                                       end_date.month, end_date.day), connect)
-    mask_prisma_2 = pd.read_sql("SELECT * FROM mask_2_params WHERE date >= '{}-{:02}-{:02}' AND  date <= \
-           '{}-{:02}-{:02}' ORDER BY date asc;".format(start_date.year, start_date.month, start_date.day, end_date.year,
-                                                       end_date.month, end_date.day), connect)
-    # amp_data_prisma_1_mask = mask_prisma_1[
-    #     ['amp1_mask', 'amp2_mask', 'amp3_mask', 'amp4_mask', 'amp5_mask', 'amp6_mask', 'amp7_mask', 'amp8_mask',
-    #      'amp9_mask', 'amp10_mask', 'amp11_mask', 'amp12_mask', 'amp13_mask',
-    #      'amp14_mask', 'amp15_mask', 'amp16_mask']]
-    amp_data_prisma_1_mask = mask_prisma_1[
-        ['amp16_mask', 'amp15_mask', 'amp14_mask', 'amp13_mask', 'amp12_mask', 'amp11_mask', 'amp10_mask', 'amp9_mask',
-         'amp8_mask', 'amp7_mask', 'amp6_mask', 'amp5_mask', 'amp4_mask',
-         'amp3_mask', 'amp2_mask', 'amp1_mask']]
-    amp_data_prisma_2_mask = mask_prisma_2[
-        ['amp16_mask', 'amp15_mask', 'amp14_mask', 'amp13_mask', 'amp12_mask', 'amp11_mask', 'amp10_mask', 'amp9_mask',
-         'amp8_mask', 'amp7_mask', 'amp6_mask', 'amp5_mask', 'amp4_mask',
-         'amp3_mask', 'amp2_mask', 'amp1_mask']]
-
-    n_data_prisma_1_mask = mask_prisma_1[
-        ['n16_mask', 'n15_mask', 'n14_mask', 'n13_mask', 'n12_mask', 'n11_mask', 'n10_mask', 'n9_mask',
-         'n8_mask', 'n7_mask', 'n6_mask', 'n5_mask', 'n4_mask',
-         'n3_mask', 'n2_mask', 'n1_mask']]
-    n_data_prisma_2_mask = mask_prisma_2[
-        ['n16_mask', 'n15_mask', 'n14_mask', 'n13_mask', 'n12_mask', 'n11_mask', 'n10_mask', 'n9_mask',
-         'n8_mask', 'n7_mask', 'n6_mask', 'n5_mask', 'n4_mask',
-         'n3_mask', 'n2_mask', 'n1_mask']]
-
-    binary_n_mask_1 = {}
-    binary_n_mask_2 = {}
-    binary_amp_mask_1 = {}
-    binary_amp_mask_2 = {}
-
-    for i in amp_data_prisma_1_mask.index:
-        amp_mask_params = amp_data_prisma_1_mask.iloc[i]
-        n_mask_params = n_data_prisma_1_mask.iloc[i]
-        binary_amp_mask_1[str(mask_prisma_1['date'][i])] = {
-            'mask_of_hit_counters_a': int(''.join(amp_mask_params.astype(str)), 2),
-            'multiplicity_of_hit_counters_a': sum(amp_mask_params)}
-
-        binary_n_mask_1[str(mask_prisma_1['date'][i])] = {
-            'mask_of_hit_counters_n': int(''.join(n_mask_params.astype(str)), 2),
-            'multiplicity_of_hit_counters_n': sum(n_mask_params)}
-
-    for i in amp_data_prisma_2_mask.index:
-        amp_mask_params = amp_data_prisma_2_mask.iloc[i]
-        n_mask_params = n_data_prisma_2_mask.iloc[i]
-        binary_amp_mask_2[str(mask_prisma_2['date'][i])] = {
-            'mask_of_hit_counters_a': int(''.join(amp_mask_params.astype(str)), 2),
-            'multiplicity_of_hit_counters_a': sum(amp_mask_params)}
-
-        binary_n_mask_2[str(mask_prisma_2['date'][i])] = {
-            'mask_of_hit_counters_n': int(''.join(n_mask_params.astype(str)), 2),
-            'multiplicity_of_hit_counters_n': sum(n_mask_params)}
-
-    return binary_n_mask_1, binary_amp_mask_1, binary_n_mask_2, binary_amp_mask_2
-
-
-if __name__ == '__main__':
-    date_time_start = datetime.date(2021, 12, 1)  # посмотреть почему не собирается конец дня 2018-04-22
-    date_time_stop = datetime.date(2021, 12, 31)
-    sqlmask_converter(date_time_start, date_time_stop)
-    print('test')

update_with_add_mask.py

@@ -1,33 +0,0 @@
-import pymongo
-import datetime
-
-from sqlmask_converter import sqlmask_converter
-from config_info.config import *
-
-db_client = pymongo.MongoClient(DB_URL)
-prisma_db = db_client["prisma-32_db"]
-
-
-def update_with_add_mask(date_, binary_n_mask_1, binary_amp_mask_1, binary_n_mask_2, binary_amp_mask_2):
-    collection_prisma = prisma_db[f'{str(date_)}_12d']
-    upd_n_result_1 = collection_prisma.update_many({'cluster': 1}, {"$set": binary_n_mask_1[str(date_)]})
-    print(f'Added n_mask_1 - {upd_n_result_1.raw_result}')
-    upd_amp_result_1 = collection_prisma.update_many({'cluster': 1}, {"$set": binary_amp_mask_1[str(date_)]})
-    print(f'Added amp_mask_1 - {upd_amp_result_1.raw_result}')
-    upd_n_result_2 = collection_prisma.update_many({'cluster': 2}, {"$set": binary_n_mask_2[str(date_)]})
-    print(f'Added n_mask_2 - {upd_n_result_2.raw_result}')
-    upd_amp_result_2 = collection_prisma.update_many({'cluster': 2}, {"$set": binary_amp_mask_2[str(date_)]})
-    print(f'Added amp_mask_2 - {upd_amp_result_2.raw_result}')
-
-
-if __name__ == '__main__':
-    date_time_start = datetime.date(2021, 12, 1)  # посмотреть почему не собирается конец дня 2018-04-22
-    date_time_stop = datetime.date(2021, 12, 31)
-    binary_n_mask_1, binary_amp_mask_1, binary_n_mask_2, binary_amp_mask_2 = sqlmask_converter(date_time_start,
-                                                                                               date_time_stop)
-    LIST_OF_DATES = [(date_time_start + datetime.timedelta(days=i)) for i in
-                     range((date_time_stop - date_time_start).days + 1)]
-    for date in LIST_OF_DATES:
-        update_with_add_mask(date, binary_n_mask_1, binary_amp_mask_1, binary_n_mask_2, binary_amp_mask_2)
-
-    print('test')