Update file path

Reviewed-on: #2

.gitignore

@@ -1 +1,2 @@
-.venv
+.venv
+.conda

Dockerfile

@@ -1,12 +1,13 @@
 # Use PHP with Apache as the base image
-FROM python:latest
+FROM python:3.9
 
 ENV PYTHONUNBUFFERED=1
 
 # Install Additional System Dependencies
 RUN apt-get update && apt-get upgrade -y
 RUN apt-get install -y \
-    libgl1 
+    libgl1 \
+    nano
 
 # Clear cache
 RUN apt-get autoremove

README.md

@@ -2,19 +2,19 @@
 
 #### Docker Build - Powershell (One Line)
 ```powershell
-$tag = ("0.0.2").Trim(); git reset --hard; git pull origin master; git tag $tag; git push --tags; docker build -t git.steelants.cz/steelants/py_ck:$tag .; docker push git.steelants.cz/steelants/py_ck:$tag; docker build -t git.steelants.cz/steelants/py_ck:latest .; docker push git.steelants.cz/steelants/py_ck:latest
+$tag = ("0.0.4").Trim(); git reset --hard; git pull origin master; git tag $tag; git push --tags; docker build -t git.steelants.cz/steelants/py_ck:$tag .; docker push git.steelants.cz/steelants/py_ck:$tag; docker build -t git.steelants.cz/steelants/py_ck:latest .; docker push git.steelants.cz/steelants/py_ck:latest
 ```
 
 #### Docker Build - Bash (One Line)
 ```bash
-export TAG="0.0.1" && sudo docker build -t git.steelants.cz/steelants/py_ck:"$TAG" . && sudo docker push git.steelants.cz/steelants/py_ck:"$TAG" && sudo docker build -t git.steelants.cz/steelants/py_ck:latest . && sudo docker push git.steelants.cz/steelants/py_ck:latest
+export TAG="0.0.5" && sudo docker build -t git.steelants.cz/steelants/py_ck:"$TAG" . && sudo docker push git.steelants.cz/steelants/py_ck:"$TAG" && sudo docker build -t git.steelants.cz/steelants/py_ck:latest . && sudo docker push git.steelants.cz/steelants/py_ck:latest
 ```
 
 #### Docker Stack Setup
 ```yaml
 services:
   ck-anonym:
-    image: git.steelants.cz/steelants/lar_py:latest
+    image: git.steelants.cz/steelants/py_ck:latest
     restart: unless-stopped
     environment:
        API_ROOT: http://ck:8080
@@ -22,4 +22,8 @@ services:
     volumes:
       - ${PWD}/app_data/:/app/data/
 networks: {}
+```
+
+```python
+pip3 install -r requirements.txt
 ```

app.py

@@ -3,69 +3,107 @@ import time
 import os
 import requests
 import json
+import gc
+import shutil
 
-treashold = float(os.getenv('FACE_TRASHOLD', '0.5'))
+from retinaface import RetinaFace
+
+treashold = float(os.getenv('FACE_TRASHOLD', '0.9'))
 sever = os.getenv('API_ROOT', 'localhost:8000')
+disk_treashold = os.getenv('DISK_FREE_TRASHOLD', 10)
+
 root_api_url = 'http://{}/local'.format(sever)
 base_path = '/app/data/'
 
 print("API_ROOT: " + str(root_api_url), flush=True)
-time.sleep(900)
+print("FACE_TRASHOLD: " + str(treashold), flush=True)
-
+print("DISK_FREE_TRASHOLD: " + str(disk_treashold), flush=True)
+    
 try:
     while True:
+        print("LOOP Started !!!", flush=True)
+        
+        #Chesk if there is enought free space at destination path
+        while True:
+            total, used, free = shutil.disk_usage(base_path)
+            if (free // (2**30)) > ((total / 100) * disk_treashold) // (2**30):
+                break
+
+            data = {"trace": "not enought free space at destination" + str(base_path) + " trashold is: " + str(disk_treashold) + "%"}
+            response = requests.post('{}/log'.format(root_api_url), json=data)
+            
+            print("sleep", flush=True)
+            time.sleep(900)
+        
+        # Load Records to be anonimized
         r = requests.get('{}/anonymize'.format(root_api_url))
-        face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
         # print(r.text, flush=True)
         response =  r.json()
 
         for detection in response:
             print(str(detection['id']) + " Anonymization Started", flush=True)
             
-            img_path = "{}{}".format(base_path, detection['source_image_path'])
+            img_path = "{}{}".format(base_path, detection['path'])
-
+            #print(img_path, flush=True)
             #Skip Non existing files
             if (not os.path.exists(img_path)):
                 continue
 
             img = cv2.imread(img_path)
-            gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+            faces = RetinaFace.detect_faces(img, treashold)
-            faces, rejects, weights = face_cascade.detectMultiScale3(gray, 1.1, 4, outputRejectLevels = 1)
 
             #Count faces  is null dont anonimize file
-            print(len(faces) + " Faces Found" , flush=True)
+            print(str(len(faces)) + " Faces Found" , flush=True)
-            if (len(faces) < 0):
+            if (len(faces) <= 0):
+                data = {
+                    "id": detection['id'],
+                    "path": img_path.replace(base_path,""),
+                    "face_found": 0,
+                }
+                response = requests.post('{}/anonymize'.format(root_api_url), json=data)
+                print(str(detection['id']) + " Anonymized 0 faces" , flush=True)
                 continue
-            
-            i = 0
-            for (x, y, w, h) in faces:
-                confidence = float(weights[i])
-                if (confidence > treashold):
-                    face_roi = img[y:y+h, x:x+w]
-                    blurred_face = cv2.GaussianBlur(face_roi, (99, 99), 50)
-                    img[y:y+h, x:x+w] = blurred_face
 
-                    print(weights[i], flush=True)
+            i = 0
-                    text = "{}".format(weights[i])
+            for (key) in faces:
-                    #cv2.rectangle(img, (x, y), (x+w, y+h), (255, 0, 0), 2)
+                confidence = faces[key]["score"]
-                    #cv2.putText(img, text, (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 2)
+                print(str(confidence) + " Faces Confidence" , flush=True)
+                x1, y1 = min(faces[key]["facial_area"][0], faces[key]["facial_area"][2]) - 2, min(faces[key]["facial_area"][1], faces[key]["facial_area"][3]) - 2
+                x2, y2 = max(faces[key]["facial_area"][0], faces[key]["facial_area"][2]) + 2, max(faces[key]["facial_area"][1], faces[key]["facial_area"][3]) + 2
+                x1, x2 = max(x1, 0), min(x2, img.shape[1])
+                y1, y2 = max(y1, 0), min(y2, img.shape[0])
+                face_roi = img[y1:y2, x1:x2]
+                blurred_face = cv2.GaussianBlur(face_roi, (99, 99), 50)
+                img[y1:y2, x1:x2] = blurred_face
+                print("Faces Anonymized" , flush=True)
 
                 i = i +1
             
+            #cv2.imshow("test_img", img)
+            #cv2.waitKey(0)
             extension = img_path[img_path.index(".")+1:]
-            new_img_path = img_path.replace((".{}".format(extension)), ('_anonym.{}'.format(extension)))
+            
+            new_img_path = img_path
+            if (not img_path.endswith(('_anonym.{}'.format(extension)))):
+                new_img_path = img_path.replace((".{}".format(extension)), ('_anonym.{}'.format(extension)))
+            else:
+                print("Path already anonymized !!!")
+                
             cv2.imwrite(new_img_path, img)
 
             data = {
                 "id": detection['id'],
-                "source_image_path": new_img_path.replace(base_path,"")
+                "path": new_img_path.replace(base_path,""),
+                "face_found": len(faces),
             }
             response = requests.post('{}/anonymize'.format(root_api_url), json=data)
             print(str(detection['id']) + " Anonymized" , flush=True)
             time.sleep(5)
         
+        print("Clearing Memory", flush=True)
+        gc.collect()
         print("sleep", flush=True)
-        time.sleep(900)
+        time.sleep(300)
 
 except Exception as e:
     print("An exception occurred", flush=True)

requirements.txt

@@ -1,2 +1,6 @@
 opencv-python
-requests
+requests
+numpy
+requests
+retina-face
+tf-keras