updating datavase compiler

src/server/cyberhex-code/api/php/database/compile_database.php

@@ -1,6 +1,6 @@
 <?php
 // Function to sort words into files based on the first two characters
-function sort_hashes($inputFile) {
+function sort_hashes($inputFile, $excluded) {
     // Open the input file for reading
     $handle = fopen($inputFile, "r");
     if ($handle === false) {
@@ -10,9 +10,16 @@ function sort_hashes($inputFile) {
     // Read each line from the input file
     while (($line = fgets($handle)) !== false) {
         // Remove leading/trailing whitespace and split the line into words
+        $line = trim($line);
+        $hash = substr($line, 0, 32); // Assuming the hash is the first 32 characters
+        
+        // Check if the hash is in the excluded array
+        if (in_array($hash, $excluded)) {
+            continue; // Skip this hash
+        }
        
         // Get the first two characters of the word
-		$prefix = substr($line, 0, 2);
+        $prefix = substr($hash, 0, 2);
         
         // Create the filename for the corresponding file
         $filename = "/var/www/html/database_srv/".$prefix . ".jdbf";
@@ -24,17 +31,17 @@ function sort_hashes($inputFile) {
         }
         
         // Write the word to the file
-		fwrite($fileHandle, $line);
+        fwrite($fileHandle, $line . PHP_EOL);
         
         // Close the file handle
         fclose($fileHandle);
-			
     }
     
     // Close the input file handle
     fclose($handle);
 }
-function download_files(){
+
+function download_files($excluded){
     //download from virusshare
     $file_count=485;
     for($i=0;$i<$file_count;$i++){
@@ -44,7 +51,7 @@ function download_files(){
         curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
         $fileContents = curl_exec($ch);
         file_put_contents("/var/www/html/database_srv/$fileNumber.md5", $fileContents);
-		sort_hashes("/var/www/html/database_srv/$fileNumber.md5");
+        sort_hashes("/var/www/html/database_srv/$fileNumber.md5", $excluded);
     }
     //download from https://bazaar.abuse.ch/export/txt/md5/recent/
     $url="https://bazaar.abuse.ch/export/txt/md5/recent/";
@@ -52,8 +59,44 @@ function download_files(){
     curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
     $fileContents = curl_exec($ch);
     file_put_contents("/var/www/html/database_srv/buf.md5", $fileContents);
-	sort_hashes("/var/www/html/database_srv/buf.md5");
+    sort_hashes("/var/www/html/database_srv/buf.md5", $excluded);
 }
+
+
+
+include "../../../config.php";
+$conn = new mysqli($DB_SERVERNAME, $DB_USERNAME, $DB_PASSWORD,$DB_DATABASE);
+if ($conn->connect_error) {
+	$success=0;
+	die("Connection failed: " . $conn->connect_error);
+}
+// Load excluded hashes from sig_ex table
+$excluded = array();
+$sql = "SELECT signature FROM sig_ex";
+$result = $conn->query($sql);
+if ($result->num_rows > 0) {
+    while($row = $result->fetch_assoc()) {
+        $excluded[] = $row["hash"];
+    }
+}
+
+// Load included hashes from sig_in table
+$included = array();
+$sql = "SELECT signature FROM sig_in";
+$result = $conn->query($sql);
+if ($result->num_rows > 0) {
+    while($row = $result->fetch_assoc()) {
+        $included[] = $row["hash"];
+    }
+}
+
+// Close the database connection
+$conn->close();
+
+// Add included array to output files this will then be processed like any other normal input file
+file_put_contents("/var/www/html/database_srv/included_hashes.txt", implode(PHP_EOL, $included));
+
+// This code updates and compiles our databases
 $directory = '/var/www/html/database_srv'; // Path to the directory
 
 // Get a list of all files in the directory
@@ -67,7 +110,5 @@ foreach ($files as $file) {
     }
 }
 set_time_limit(0);
-download_files();
-
-
+download_files($excluded);
 ?>

src/server/cyberhex-code/system/secure_zone/php/database_settings.php

@@ -123,7 +123,7 @@ async function add_item(db,element_id1,field1,element_id2,field2){ //we have two
 						<a class="nav-link" href="database_settings.php?show=excluded" id="excluded_tab">Excluded signatures</a>
 					  </li>
 					  <li class="nav-item">
-						<a class="nav-link" href="database_settings.php?show=excluded" id="included_tab">Included Signatures</a>
+						<a class="nav-link" href="database_settings.php?show=included" id="included_tab">Included Signatures</a>
 					  </li>
 
 				</ul>