Changeset 1558

Timestamp:

02/19/08 00:27:59 (6 months ago)

Author:

till

Message:

nebula
- first daemon version

Files:

• nebula/trunk/Makefile.am (modified) (1 diff)
• nebula/trunk/configure.in (modified) (1 diff)
• nebula/trunk/gst (deleted)
• nebula/trunk/src/Makefile.am (modified) (2 diffs)
• nebula/trunk/src/Makefile.in (added)
• nebula/trunk/src/avl.c (added)
• nebula/trunk/src/avl.h (added)
• nebula/trunk/src/classify.c (added)
• nebula/trunk/src/classify.h (added)
• nebula/trunk/src/cluster.c (modified) (3 diffs)
• nebula/trunk/src/cluster.h (modified) (2 diffs)
• nebula/trunk/src/cstr.c (added)
• nebula/trunk/src/cstr.h (added)
• nebula/trunk/src/hash.c (modified) (2 diffs)
• nebula/trunk/src/hash.h (modified) (1 diff)
• nebula/trunk/src/lca.c (added)
• nebula/trunk/src/lca.h (added)
• nebula/trunk/src/nebula.c (modified) (8 diffs)
• nebula/trunk/src/nebula.h (modified) (1 diff)
• nebula/trunk/src/nebulad.c (added)
• nebula/trunk/src/net.c (added)
• nebula/trunk/src/net.h (added)
• nebula/trunk/src/queue.c (modified) (11 diffs)
• nebula/trunk/src/queue.h (modified) (2 diffs)
• nebula/trunk/src/session.c (added)
• nebula/trunk/src/session.h (added)
• nebula/trunk/src/sig.c (added)
• nebula/trunk/src/sig.h (added)
• nebula/trunk/src/signals.c (modified) (1 diff)
• nebula/trunk/src/stree.c (added)
• nebula/trunk/src/stree.h (added)
• nebula/trunk/src/util.c (modified) (2 diffs)
• nebula/trunk/src/util.h (modified) (2 diffs)

nebula/trunk/Makefile.am

@@ -1 +1 @@
-AUTOMAKE_OPTIONS = foreign
-
- gst
+
-
-EXTRA_DIST = ChangeLog LICENSE README

nebula/trunk/configure.in

@@ -37 +37 @@
-AC_PROG_INSTALL
-AC_CONFIG_FILES([Makefile
-
-
+
-
-AC_OUTPUT

nebula/trunk/src/Makefile.am

@@ -1 +1 @@
-AM_CFLAGS=-Wall -Werror
-
-
+ -lz -lpthread
-
-bin_PROGRAMS = nebula
@@ -12 +12 @@
-                        hash.c hash.h \
-                        cluster.c cluster.h \
+                        session.c session.h \
+                        net.c net.h \
+                        classify.c classify.h \
+                        avl.c avl.h \
+                        lca.c lca.h \
+                        sig.c sig.h \
+                        cstr.c cstr.h \
+                        stree.c stree.h \
-                        nebula.c nebula.h

nebula/trunk/src/cluster.c

@@ -48 +48 @@
-
-/* delete cluster */
-, u_char list_flag
+
-        if (!cl) return;
-
-
-
+
-        free(cl);
-
@@ -71 +70 @@
-                        exit(EXIT_FAILURE);
-                }
+                cl->cnt++;
-        } else {
-                if (verbose) fprintf(stderr, "Warning - Could not add hash to cluster queue: Maximum size reached.\n");
@@ -103 +103 @@
-
-                // unlink and free old cluster
-0, 
+
-                entry = queue_unlink(q, src->parent);
-, 1
+
-                free(entry);
-

nebula/trunk/src/cluster.h

@@ -41 +41 @@
-        struct cluster *next;
-        queue *hq;
+
-} cluster;
-
-
-
-inline cluster *cluster_new(void);
-, u_char list_flag
+
-
-inline cluster *create_hashlist(hash *hl1, hash *hl2);
@@ -52 +52 @@
-cluster *create_cluster(hash *l1, hash *l2);
-cluster *clusters_merge(queue *q, cluster *dst, cluster *src);
-, u_char list_files
+
-
-#endif

nebula/trunk/src/hash.c

@@ -26 +26 @@
-#include "hash.h"
-#include "nebula.h"
+#include "session.h"
-
-hash *hash_new(void) {
@@ -37 +38 @@
-
-
-, u_char list_flag
+
-        if (!h) return;
-
-        if (list_flag && ((hash *)h)->filename) printf("    %s\n", ((hash *)h)->filename);
-        free(((hash *)h)->md5sum);
+        ((hash *)h)->md5sum = NULL;
+        ((submission *)((hash *)h)->submission)->md5sum = NULL;
+
-        free(((hash *)h)->spamsum);
-
+
+
+
+
+
+
+
+
-
-        if (((hash *)h)->free) ((hash *)h)->free(h);

nebula/trunk/src/hash.h

@@ -30 +30 @@
-
-typedef struct hash {
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
-} hash;
-
-
-hash *hash_new(void);
-, u_char list_flag
+
-
-#endif

nebula/trunk/src/nebula.c

@@ -23 +23 @@
-#include <fcntl.h>
-#include <math.h>
+#include <poll.h>
+#include <pthread.h>
-#include <stdlib.h>
-#include <stdio.h>
@@ -32 +34 @@
-#include <unistd.h>
-
+#include "classify.h"
-#include "cluster.h"
-#include "md5.h"
-#include "nebula.h"
+#include "net.h"
-#include "ngram.h"
+#include "session.h"
+#include "sig.h"
-#include "signals.h"
-#include "spamsum.h"
@@ -45 +51 @@
-
-
+#define POLLFD_SET_SIZE 10
+
+
-void usage(const char* progname, const int exit_val) {
-        printf("Usage: %s "
-                " -C <size>\t cluster queue size\n"
+                "\t\t -c <similarity> cluster criteria (a similarity measure in percent)\n"
+                "\t\t -d\t\t daemonize\n"
-                "\t\t -E <size>\t cluster element queue size\n"
+                "\t\t -h\t\t show this help\n"
-                "\t\t -O <size>\t outlier queue size\n"
-
-
-
-
-
-
-
-
-
-
-
+
+
-                "\t\t -v\t\t be verbose\n"
-                "\t\t\t\t [files ...]\n",
@@ -69 +72 @@
-
-int main(int argc, char *argv[]) {
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-
-        outlierq                = NULL;
-        clusterq                = NULL;
-
-
-
-
+
+
-        i                       = 0;
-        qsize                   = 0;
-        total_files             = 0;
-#ifdef PROFILE
-        alarm_time              = 5;
-#endif
-
-        /* default values for parameters */
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-
-        memset(&md5sum_trie, 0, sizeof(trie_node));
-        memset(&spamsum_trie, 0, sizeof(trie_node));
-
-
+
+
+
+
+
-
-        // process args
-
-
-
-
-
+
-                switch(option) {
-#ifdef PROFILE
-                        case 'a':
-                                alarm_time = atoi(optarg);
-                                if (alarm_time < 0) {
-                                        fprintf(stderr, "Error - Profile time interval must be a non-negative value (0 means 'off').\n");
-                                        exit(EXIT_FAILURE);
-                                }
-                                break;
-#endif
-                        case 'c':
-                                cluster_radius = atof(optarg);
@@ -145 +129 @@
-                                break;
-                        case 'd':
-
-
-
-
+
-                                break;
-                        case 'E':
@@ -157 +138 @@
-                                }
-                                break;
-                        case 'l':
-                                list_files = 1;
-                                break;
-                        case 'o':
-                                list_outlier = 1;
-                                break;
-                        case 'O':
-                                outlierq_max = atoi(optarg);
@@ -171 +146 @@
-                                break;
-                        case 'p':
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-                                break;
-                        case 'v':
- = 1
+++
-                                break;
-                        case 'h':
@@ -190 +166 @@
-        }
-
-        if (hide_result && list_files) list_files = 0;
-
-        set_signal_handlers();
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                        }
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                        }
-                }
-                total_files = n;
-        } else {
-                if ((argc - optind) < 1) usage(argv[0], EXIT_FAILURE);
-                i = optind;
-                total_files = argc - i;
-                if (time_sort) printf("Only directories (-d) can be processed chronologically (-t).\n");
-        }
-
-        // initialize outlier queue
-        outlierq = queue_new();
-        clusterq = queue_new();
-
-
-#ifdef PROFILE
-        if (alarm_time) {
-                alarm(alarm_time);
-                printf("Profiling enabled, printing statistics every %d seconds.\n", alarm_time);
-                files_in_interval = 0;
-                bytes_in_interval = 0;
-                checkpoint = 0;
-        }
-#endif
-
-        printf("processing %u files.\n", (unsigned int) total_files);
-        if (!verbose && show_progress) {
-                printf("files processed:                       ");
-                fflush(stdout);
-        }
-
-
-        for (n=0; n<total_files; n++) {
-                filename = (dirname ? namelist[n]->d_name : argv[n+i]);
-
-                bstr = bstr_map(filename);
-                num_of_files++;
-#ifdef PROFILE
-                files_in_interval++;
-                bytes_in_interval += bstr.len;
-#endif
-
-                if (verbose) printf("  processing file %s.\n", filename);
-                else if (show_progress) {
-                        printf("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b");
-                        if (num_of_files < total_files)
-                                printf("files processed: %05.2f%% (%04u clusters)", num_of_files/total_files*100, (unsigned int) clusterq->size);
-                        else
-                                printf("files processed: 100.00%% (%04u clusters)\n", (unsigned int) clusterq->size);
-                        fflush(stdout);
-                }
-
-
-                // calculate md5sum
-                if ((tmpbuf = (u_char *) mem_md5sum(bstr.data, bstr.len)) == NULL) {
-                        fprintf(stderr, "Error - Unable to allocate memory: %m.\n");
-                        exit(EXIT_FAILURE);
-                }
-
-                if ((t = trie_find_memstr(&md5sum_trie, tmpbuf, strlen((char *)tmpbuf))) != NULL) {
-                   
-                        // md5sum is already in trie
-                        free(tmpbuf);
-                        ((hash*)t->data)->cnt++;
-                        if (verbose) printf("    md5sum is %s (%u instances)\n", ((hash*)t->data)->md5sum, ((hash*)t->data)->cnt);
-                        if (verbose) printf("    absolute match found.\n");
-                        num_of_duplicates++;
-                } else {
-                        // md5sum not in trie, create new element
-                        t = trie_memins(&md5sum_trie, tmpbuf, strlen((char *)tmpbuf), NULL);
-                        if ((t->data = calloc(1, sizeof(hash))) == NULL) {
-                                fprintf(stderr, "Error - Unable to allocate memory: %m.\n");
-                                exit(EXIT_FAILURE);
-                        }
-                        ((hash*)t->data)->hashlen = 32;
-                        ((hash*)t->data)->md5sum = (char *) tmpbuf;
-                        ((hash*)t->data)->cnt++;
-                        // set filename
-                        if ((((hash*)t->data)->filename = strdup(filename)) == NULL) {
-                                fprintf(stderr, "Error - Unable to allocate memory: %m.\n");
-                                exit(EXIT_FAILURE);
-                        }
-                        // set spamsum hash
-                        if ((((hash*)t->data)->spamsum = spamsum(bstr.data, bstr.len, 0)) == NULL) {
-                                fprintf(stderr, "Error - Unable to allocate memoory: %m.\n");
-                                exit(EXIT_FAILURE);
-                        }
-                        if (verbose) printf("    md5sum is %s (%u instances)\n", ((hash*)t->data)->md5sum, ((hash*)t->data)->cnt);
-
-                        // connect all clusters within range
-                        for (cur_cqelem = clusterq->head; cur_cqelem; cur_cqelem = cur_cqelem->next) {
-                                for (cur_hqelem = ((cluster *)cur_cqelem->data)->hq->head; cur_hqelem; cur_hqelem = cur_hqelem->next) {
-                                        if ((score = spamsum_match(((hash*)t->data)->spamsum, ((hash*)cur_hqelem->data)->spamsum)) >= cluster_radius) {
-                                                if (!((hash*)t->data)->cl) {
-                                                        add_entry_to_cluster((cluster *)cur_cqelem->data, (hash*)t->data);
-                                                        break;
-                                                } else {
-                                                        if ((cluster *) cur_cqelem->data != ((hash*)t->data)->cl)
-                                                                clusters_merge(clusterq, (cluster *) cur_cqelem->data, ((hash*)t->data)->cl);
-                                                        break;
-                                                }
-                                        }
-                                }
-                        }
-
-                        // connect all outliers within range
-                        for (cur_hqelem = outlierq->head; cur_hqelem; cur_hqelem = cur_hqelem->next) {
-                                if ((score = spamsum_match(((hash*)t->data)->spamsum, ((hash*)cur_hqelem->data)->spamsum)) >= cluster_radius) {
-                                        // unlink match from outlier list
-                                        tmp_hqelem = cur_hqelem;
-                                        cur_hqelem = cur_hqelem->next;
-                                        if ((tmp_hash = queue_unlink(outlierq, tmp_hqelem)) == NULL) {
-                                                fprintf(stderr, "Error - Unable to unlink outlier from queue.\n");
-                                                exit(EXIT_FAILURE);
-                                        }
-                                        if (((hash*)t->data)->cl) {
-                                                // add other outliers to cluster
-                                                if (add_entry_to_cluster(((hash*)t->data)->cl, tmp_hash) == NULL) {
-                                                        trie_del_memstr(&md5sum_trie, (u_char *) tmp_hash->md5sum, strlen(tmp_hash->md5sum));
-                                                        trie_del_memstr(&spamsum_trie, (u_char *) tmp_hash->spamsum, strlen(tmp_hash->spamsum));
-                                                        hash_free(tmp_hash, 0);
-                                                }
-                                        } else {
-                                                // create new cluster of two outliers
-                                                if ((tmp_cqelem = queue_ins(clusterq, create_cluster(((hash*)t->data), tmp_hash), clusterq_max)) != NULL) {
-                                                        /* cluster queue is full, last element was dropped and must be free()d */
-                                                        
-                                                        /* first remove hashes from tries */
-                                                        for (tmp_hqelem = ((cluster *)tmp_cqelem->data)->hq->head; tmp_hqelem; tmp_hqelem = tmp_hqelem->next) {
-                                                                trie_del_memstr(&md5sum_trie, (u_char *) ((hash *)tmp_hqelem->data)->md5sum,
-                                                                        strlen(((hash *)tmp_hqelem->data)->md5sum));
-                                                                trie_del_memstr(&spamsum_trie, (u_char *) ((hash *)tmp_hqelem->data)->spamsum,
-                                                                        strlen(((hash *)tmp_hqelem->data)->spamsum));
-                                                        }
-                                                        /* now free cluster */
-                                                        cluster_free((cluster *)tmp_cqelem->data, 0);
-                                                        free(tmp_cqelem);
-                                                }
-                                                /* set cluster element's parent pointer to cluster queue head
-                                                 * we need this to be able to unlink a cluster from the queue */
-                                                ((cluster *) clusterq->head->data)->parent = clusterq->head;
-                                        }
-                                }
-                                if (!cur_hqelem) break;
-                        }
-
-                                
-                        if (verbose) printf("    spamsum is %s (%u instances)\n", ((hash*)t->data)->spamsum, ((hash*)t->data)->cnt);
-
-                        if (!((hash*)t->data)->cl) {
-                                // insert outlier into queue
-                                if (outlierq->size >= outlierq_max) {
-                                        tmp_hash = queue_unlink(outlierq, outlierq->tail);
-                                        trie_del_memstr(&md5sum_trie, (u_char *) tmp_hash->md5sum, strlen(tmp_hash->md5sum));
-                                        trie_del_memstr(&spamsum_trie, (u_char *) tmp_hash->spamsum, strlen(tmp_hash->spamsum));
-                                        hash_free(tmp_hash, 0);
-                                }
-                                queue_ins(outlierq, t->data, outlierq_max);
-                        }
-                }
-                bstr_unmap(bstr);
-                if (verbose) printf("\n");
-        }
-
-        cleanup();

nebula/trunk/src/nebula.h

@@ -27 +27 @@
-
-#include <sys/types.h>
+#include <pthread.h>
-
+#include "avl.h"
-#include "cluster.h"
-#include "hash.h"
+#include "stree.h"
-
-
-
+
+
+
+
-ssize_t         clusterq_max, clusterhashq_max, outlierq_max; 
-u_int32_t       num_of_duplicates;
-float           num_of_files, total_files;
-double          cluster_radius;
-
+
-queue           *clusterq;
-queue           *outlierq;
+queue           *aconnq;
-
-
-
-
-
-
-
+
+
-
-#endif

nebula/trunk/src/queue.c

@@ -30 +30 @@
-        qelem *new;
-        
-
+
+
+
-
-        if ((new = calloc(1, sizeof(qelem))) == NULL) return(NULL);
@@ -44 +46 @@
-        q->size++;
-
+        pthread_rwlock_unlock(&q->lock);
-        return(new);
-}
@@ -51 +54 @@
-        qelem *new;
-        
-
+
+
+
-
-        if ((new = calloc(1, sizeof(qelem))) == NULL) return(NULL);
@@ -65 +70 @@
-        q->size++;
-
+        pthread_rwlock_unlock(&q->lock);
-        return(new);
-}
@@ -72 +78 @@
-        qelem *tmp;
-
-
+
+
+
-
-        tmp = q->head;
@@ -80 +88 @@
-        q->size--;
-
+        pthread_rwlock_unlock(&q->lock);
-        return(tmp);
-}
@@ -87 +96 @@
-        qelem *tmp;
-
-
+
+
+
-
-        tmp = q->tail;
@@ -95 +106 @@
-        q->size--;
-
+        pthread_rwlock_unlock(&q->lock);
-        return(tmp);
-}
@@ -102 +114 @@
-        qelem* tmp = NULL;
-
-
+q || !
-
-        /* need to cut off last queue element? */
@@ -124 +136 @@
-                e = queue_cuttail(q);
-        } else {
+                pthread_rwlock_wrlock(&q->lock);
+
-                e->prev->next = e->next;
-                e->next->prev = e->prev;
-                if (!q->size--) q->head = q->tail = NULL;
+
+                pthread_rwlock_unlock(&q->lock);
-        }
-
@@ -142 +158 @@
-                exit(EXIT_FAILURE);
-        }
+        if (pthread_rwlock_init(&q->lock, NULL) != 0) {
+                fprintf(stderr, "Error - Unable to initialize queue mutex: %m.\n");
+                exit(EXIT_FAILURE);
+        }
+
-        return(q);
-}
-
-
-u_char list_flag, void(*cbfn)(void *data, u_char list_flag
+void(*cbfn)(void *data
-        qelem *cur;
-
-        if (!q) return;
-