Skip to content

Commit 3952e1b

Browse files
alainabbasalainabbas
committed
primaryKey
1 parent eae6cf4 commit 3952e1b

File tree

7 files changed

+24
-246
lines changed

7 files changed

+24
-246
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@ src/__pycache__/*
77
lib/
88
bin/
99
pyvenv.cfg
10+
config.yml

‎Makefile‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ run-crawler-docker: ## Lance le crawler Sesame - Taiga avec python !
2828
--platform $(PLATFORM) \
2929
--name $(APP_NAME) \
3030
-v $(CURDIR):/data \
31-
sesame-taiga_crawler
31+
ghcr.io/libertech-fr/sesame-crawler:latest
3232

3333
run-crawler: ## Lance le crawler Sesame - Taiga avec python !
3434
@python3 main.py
@@ -38,14 +38,6 @@ install-deps: ## Installe les dépendances python
3838
@pip install -r requirements.txt
3939
@printf "\033[33mPIP:\033[0m SUCCESSFUL !!!\n"
4040

41-
taiga-forward: ## Transfert les appels de l'API Taiga via un proxy socks au travers du serveur sesame (à utiliser pour lancer le script à distance)
42-
@printf "\033[33mNCAT:\033[0m Launch forwarding tcp requests for <$(STC_API_HOST)> ...\n"
43-
@ssh libertech@$(STC_API_TARGET) "pkill -f 'ncat $(STC_API_HOST) 443'" || true
44-
@ssh libertech@$(STC_API_TARGET) "ncat --keep-open --no-shutdown -v \
45-
--sh-exec 'ncat $(STC_API_HOST) 443' \
46-
-l $(STC_API_FORWARD_PORT)"
47-
@printf "\033[33mNCAT:\033[0m End of forwarding requests !\n"
48-
4941
update-reqs: ## Met à jour la liste des dépendances python
5042
@printf "\033[33mUPDATE:\033[0m pipreqs dependency ...\n"
5143
@pip install pipreqs > ./logs/update-reqs.log

‎config.yml‎

Lines changed: 5 additions & 156 deletions
Original file line numberDiff line numberDiff line change
@@ -1,157 +1,6 @@
1-
---
2-
taiga_etd.json:
1+
import.json:
2+
primaryKey:
3+
- inetOrgPerson.uid
34
mapping:
4-
inetOrgPerson.cn:
5-
- "nom"
6-
- "prenom"
7-
inetOrgPerson.displayName:
8-
- "prenom"
9-
- "nom"
10-
inetOrgPerson.sn: "nom"
11-
inetOrgPerson.givenName: "prenom"
12-
additionalFields.attributes.supannPerson.supannPrenomsEtatCivil: "prenom"
13-
inetOrgPerson.employeeNumber: "id_coord"
14-
"$setOnInsert.inetOrgPerson.uid":
15-
- "prenom"
16-
- "nom"
17-
additionalFields.attributes.supannPerson.supannRefId: "id_coord"
18-
"$setOnInsert.inetOrgPerson.mail":
19-
- "prenom"
20-
- "nom"
21-
inetOrgPerson.mobile: "tel_mob"
22-
inetOrgPerson.postalAddress:
23-
- "adresse"
24-
- "CP"
25-
- "Ville"
26-
additionalFields.attributes.supannPerson.supanncivilite: "civilite"
27-
additionalFields.attributes.supannPerson.supannOIDCGenre: "civilite"
28-
additionalFields.attributes.supannPerson.supannNomdeNaissance: "nom_marital"
29-
additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance: "nss_date"
30-
additionalFields.attributes.supannPerson.supannAutreMail: "email2"
31-
additionalFields:
32-
additionalFields.objectClasses:
33-
- "supannPerson"
34-
additionalFields.attributes.supannPerson.supannTypeEntiteAffectation: "etd"
35-
state: -1
36-
inetOrgPerson.employeeType: "TAIGA"
37-
inetOrgPerson.departmentNumber: "etd"
38-
additionalFields.attributes.supannPerson.supannEtablissement: "{UAI}0690184A"
39-
transforms:
40-
inetOrgPerson.cn: "join(delimiter=' ')"
41-
inetOrgPerson.displayName: "join(delimiter=' ')"
42-
inetOrgPerson.postalAddress: "join(delimiter=',')"
43-
"$setOnInsert.inetOrgPerson.uid":
44-
- "join(delimiter='.')"
45-
- "remove_accents"
46-
- "lower"
47-
- "replace(old=' ',new='-')"
48-
# - "regex(pattern='(?<=\\b\\w)([a-zA-Z0-9_\\-]+\\.)', replace='')"
49-
"$setOnInsert.inetOrgPerson.mail":
50-
- "join(delimiter='.')"
51-
- "lower"
52-
- "remove_accents"
53-
- "replace(old=' ',new='-')"
54-
- "suffix(string='@sesame.local')"
55-
# taiga_adm.json:
56-
# mapping:
57-
# inetOrgPerson.cn:
58-
# - "nom"
59-
# - "prenom"
60-
# inetOrgPerson.displayName:
61-
# - "prenom"
62-
# - "nom"
63-
# inetOrgPerson.sn: "nom"
64-
# inetOrgPerson.givenName: "prenom"
65-
# additionalFields.attributes.supannPerson.supannPrenomsEtatCivil: "prenom"
66-
# inetOrgPerson.employeeNumber: "id_coord"
67-
# $setOnInsert.inetOrgPerson.uid:
68-
# - "prenom"
69-
# - "nom"
70-
# additionalFields.attributes.supannPerson.supannEmpId: "id_coord"
71-
# $setOnInsert.inetOrgPerson.mail:
72-
# - "prenom"
73-
# - "nom"
74-
# inetOrgPerson.mobile: "tel_mob"
75-
# inetOrgPerson.postalAddress:
76-
# - "adresse"
77-
# - "CP"
78-
# - "Ville"
79-
# additionalFields.attributes.supannPerson.supanncivilite: "civilite"
80-
# additionalFields.attributes.supannPerson.supannOIDCGenre: "civilite"
81-
# additionalFields.attributes.supannPerson.supannNomdeNaissance: "nom_marital"
82-
# additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance: "nss_date"
83-
# additionalFields.attributes.supannPerson.supannAutreMail: "email2"
84-
# additionalFields:
85-
# additionalFields.objectClasses:
86-
# - "supannPerson"
87-
# additionalFields.attributes.supannPerson.supannTypeEntiteAffectation: "adm"
88-
# inetOrgPerson.employeeType: "TAIGA"
89-
# inetOrgPerson.departmentNumber: "adm"
90-
# state: -1
91-
# transforms:
92-
# inetOrgPerson.cn: "join(delimiter=' ')"
93-
# inetOrgPerson.displayName: "join(delimiter=' ')"
94-
# inetOrgPerson.postalAddress: "join(delimiter=',')"
95-
# $setOnInsert.inetOrgPerson.uid:
96-
# - "join(delimiter='.')"
97-
# - "remove_accents"
98-
# - "lower"
99-
# - "replace(old=' ',new='-')"
100-
# # - "regex(pattern='(?<=\\b\\w)([a-zA-Z0-9_\\-]+\\.)', replace='')"
101-
# $setOnInsert.inetOrgPerson.mail:
102-
# - "join(delimiter='.')"
103-
# - "remove_accents"
104-
# - "lower"
105-
# - "replace(old=' ',new='-')"
106-
# - "suffix(string='@sesame.local')"
107-
# taiga_esn.json:
108-
# mapping:
109-
# inetOrgPerson.cn:
110-
# - "nom"
111-
# - "prenom"
112-
# inetOrgPerson.displayName:
113-
# - "prenom"
114-
# - "nom"
115-
# inetOrgPerson.sn: "nom"
116-
# inetOrgPerson.givenName: "prenom"
117-
# additionalFields.attributes.supannPerson.supannPrenomsEtatCivil: "prenom"
118-
# inetOrgPerson.employeeNumber: "id_coord"
119-
# $setOnInsert.inetOrgPerson.uid:
120-
# - "prenom"
121-
# - "nom"
122-
# additionalFields.attributes.supannPerson.supannEmpId: "id_coord"
123-
# $setOnInsert.inetOrgPerson.mail:
124-
# - "prenom"
125-
# - "nom"
126-
# inetOrgPerson.mobile: "tel_mob"
127-
# inetOrgPerson.postalAddress:
128-
# - "adresse"
129-
# - "CP"
130-
# - "Ville"
131-
# additionalFields.attributes.supannPerson.supanncivilite: "civilite"
132-
# additionalFields.attributes.supannPerson.supannOIDCGenre: "civilite"
133-
# additionalFields.attributes.supannPerson.supannNomdeNaissance: "nom_marital"
134-
# additionalFields.attributes.supannPerson.supannOIDCDatedeNaissance: "nss_date"
135-
# additionalFields.attributes.supannPerson.supannAutreMail: "email2"
136-
# additionalFields:
137-
# additionalFields.objectClasses:
138-
# - "supannPerson"
139-
# additionalFields.attributes.supannPerson.supannTypeEntiteAffectation: "esn"
140-
# inetOrgPerson.departmentNumber: "esn"
141-
# inetOrgPerson.employeeType: "TAIGA"
142-
# state: -1
143-
# transforms:
144-
# inetOrgPerson.cn: "join(delimiter=' ')"
145-
# inetOrgPerson.displayName: "join(delimiter=' ')"
146-
# inetOrgPerson.postalAddress: "join(delimiter=',')"
147-
# $setOnInsert.inetOrgPerson.uid:
148-
# - "join(delimiter='.')"
149-
# - "remove_accents"
150-
# - "lower"
151-
# # - "regex(pattern='(?<=\\b\\w)([a-zA-Z0-9_\\-]+\\.)', replace='')"
152-
# $setOnInsert.inetOrgPerson.mail:
153-
# - "join(delimiter='.')"
154-
# - "remove_accents"
155-
# - "lower"
156-
# - "replace(old=' ',new='-')"
157-
# - "suffix(string='@sesame.local')"
5+
inetOrgPerson.uid: uid
6+
inetOrgPerson.title: title

‎main.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@
66
import hashlib
77
from datetime import datetime
88

9-
from src.a_moins_b import a_moins_b
10-
from src.export_ind import export_ind
9+
1110
from src.import_ind import import_ind
1211

1312
logging.basicConfig(level=logging.INFO)

‎src/a_moins_b.py‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎src/export_ind.py‎

Lines changed: 0 additions & 43 deletions
This file was deleted.

‎src/import_ind.py‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,17 @@ async def read_response(response):
2727
jsonMessage = json.loads(message)
2828
print(jsonMessage)
2929

30-
async def send_request(session, url, json):
30+
async def send_request(session, url, json, primaryKey):
3131
headers = {
3232
"Authorization": f"Bearer {sesame_api_token}",
3333
"Content-Type": "application/json; charset=utf-8",
3434
}
35-
params = {
36-
"filters[inetOrgPerson.employeeNumber]": f"{json.get('inetOrgPerson', {}).get('employeeNumber')}",
37-
"filters[inetOrgPerson.employeeType]": "TAIGA",
38-
}
35+
36+
params = {}
37+
for k in primaryKey:
38+
mykey="filters[" + k + "]"
39+
params[mykey]=get_value(json,k)
40+
3941

4042
try:
4143

@@ -69,7 +71,7 @@ async def process_data(data, config, file, session):
6971
result = await get_data(data, config)
7072
with open(f'./data/{file}', 'w', encoding='utf-8') as fichier:
7173
json.dump(result, fichier, ensure_ascii=False, indent=4)
72-
tasks = [send_request(session, f'{sesame_api_baseurl}/management/identities/upsert', entry) for entry in result]
74+
tasks = [send_request(session, f'{sesame_api_baseurl}/management/identities/upsert', entry,config.get('primaryKey',["inetOrgPerson.employeeType","inetOrgPerson.employeeNumber"])) for entry in result]
7375
await gather_with_concurrency(sesame_import_parallels_files, tasks)
7476
print(f"Processed {file}")
7577

@@ -90,3 +92,11 @@ async def import_ind():
9092
async with aiohttp.ClientSession() as session:
9193
tasks = [process_data(datas[file], configs[file], file, session) for file in cache_files if file in configs.keys()]
9294
await gather_with_concurrency(sesame_import_parallels_files, tasks)
95+
96+
97+
def get_value(data, path):
98+
keys = path.split('.')
99+
value = data
100+
for key in keys:
101+
value = value[key]
102+
return value

0 commit comments

Comments
 (0)