| 82 | | if (t != NULL) { |
|---|
| 83 | | // md5sum is already in trie |
|---|
| 84 | | ((hash*)t->data)->cnt++; |
|---|
| 85 | | |
|---|
| 86 | | if (verbose > 1) printf(" MD5 hash is %s (%u instances)\n", ((hash*)t->data)->md5sum, ((hash*)t->data)->cnt); |
|---|
| 87 | | if (verbose) printf(" Absolute MD5 match found.\n"); |
|---|
| 88 | | return(0); |
|---|
| 89 | | } else { |
|---|
| 90 | | // md5sum not in trie, create new element |
|---|
| 91 | | pthread_rwlock_wrlock(&md5sum_trie_lock); |
|---|
| 92 | | t = trie_memins(&md5sum_trie, (u_char *) s->md5sum, strlen(s->md5sum), NULL); |
|---|
| 93 | | pthread_rwlock_unlock(&md5sum_trie_lock); |
|---|
| 94 | | |
|---|
| 95 | | if ((t->data = calloc(1, sizeof(hash))) == NULL) { |
|---|
| 96 | | fprintf(stderr, "Error - Unable to allocate memory: %m.\n"); |
|---|
| 97 | | exit(EXIT_FAILURE); |
|---|
| 98 | | } |
|---|
| 99 | | ((hash*)t->data)->hashlen = 32; |
|---|
| 100 | | ((hash*)t->data)->md5sum = s->md5sum; |
|---|
| 101 | | |
|---|
| 102 | | if ((((hash*)t->data)->submission = calloc(1, sizeof(submission))) == NULL) { |
|---|
| 103 | | fprintf(stderr, "Error - Unable to allocate memory: m.\n"); |
|---|
| 104 | | exit(EXIT_FAILURE); |
|---|
| 105 | | } |
|---|
| 106 | | memcpy(((hash*)t->data)->submission, s, sizeof(submission)); |
|---|
| 107 | | ((hash*)t->data)->cnt++; |
|---|
| 108 | | |
|---|
| 109 | | // set spamsum hash |
|---|
| 110 | | if ((((hash*)t->data)->spamsum = spamsum(s->attack, s->attack_len, 0)) == NULL) { |
|---|
| 111 | | fprintf(stderr, "Error - Unable to allocate memoory: %m.\n"); |
|---|
| 112 | | exit(EXIT_FAILURE); |
|---|
| 113 | | } |
|---|
| 114 | | if (verbose > 1) printf(" MD5 hash is %s (%u instances)\n", ((hash*)t->data)->md5sum, ((hash*)t->data)->cnt); |
|---|
| 115 | | |
|---|
| 116 | | |
|---|
| 117 | | Q_WLOCK(&clusterq->lock); |
|---|
| 118 | | |
|---|
| 119 | | // connect all clusters within range |
|---|
| 120 | | for (cur_cqelem = clusterq->head; cur_cqelem; cur_cqelem = cur_cqelem->next) { |
|---|
| 121 | | for (cur_hqelem = ((cluster *)cur_cqelem->data)->hq->head; cur_hqelem; cur_hqelem = cur_hqelem->next) { |
|---|
| 122 | | if ((score = spamsum_match(((hash*)t->data)->spamsum, ((hash*)cur_hqelem->data)->spamsum)) >= cluster_radius) { |
|---|
| 123 | | if (!((hash*)t->data)->cl) { |
|---|
| 124 | | add_entry_to_cluster((cluster *)cur_cqelem->data, (hash*)t->data); |
|---|
| 125 | | if (verbose) printf(" Cluster has now %u elements (threshold is %lu).\n", |
|---|
| 126 | | ((hash*)t->data)->cl->cnt, ((hash*)t->data)->cl->threshold); |
|---|
| 127 | | break; |
|---|
| 128 | | } else { |
|---|
| 129 | | if ((cluster *) cur_cqelem->data != ((hash*)t->data)->cl) |
|---|
| 130 | | clusters_merge(clusterq, (cluster *) cur_cqelem->data, ((hash*)t->data)->cl); |
|---|
| 131 | | break; |
|---|
| 132 | | } |
|---|
| 133 | | } else if (score > max_score) max_score = score; |
|---|
| 134 | | } |
|---|
| 135 | | } |
|---|
| 136 | | |
|---|
| 137 | | // connect all outliers within range |
|---|
| 138 | | for (cur_hqelem = outlierq->head; cur_hqelem; cur_hqelem = cur_hqelem->next) { |
|---|
| | 84 | if ((t->data = calloc(1, sizeof(hash))) == NULL) { |
|---|
| | 85 | fprintf(stderr, "Error - Unable to allocate memory: %m.\n"); |
|---|
| | 86 | exit(EXIT_FAILURE); |
|---|
| | 87 | } |
|---|
| | 88 | ((hash*)t->data)->hashlen = 32; |
|---|
| | 89 | ((hash*)t->data)->md5sum = s->md5sum; |
|---|
| | 90 | |
|---|
| | 91 | if ((((hash*)t->data)->submission = calloc(1, sizeof(submission))) == NULL) { |
|---|
| | 92 | fprintf(stderr, "Error - Unable to allocate memory: m.\n"); |
|---|
| | 93 | exit(EXIT_FAILURE); |
|---|
| | 94 | } |
|---|
| | 95 | memcpy(((hash*)t->data)->submission, s, sizeof(submission)); |
|---|
| | 96 | ((hash*)t->data)->cnt++; |
|---|
| | 97 | |
|---|
| | 98 | // set spamsum hash |
|---|
| | 99 | if ((((hash*)t->data)->spamsum = spamsum(s->attack, s->attack_len, 0)) == NULL) { |
|---|
| | 100 | fprintf(stderr, "Error - Unable to allocate memoory: %m.\n"); |
|---|
| | 101 | exit(EXIT_FAILURE); |
|---|
| | 102 | } |
|---|
| | 103 | if (verbose > 1) printf(" MD5 hash is %s (%u instances)\n", ((hash*)t->data)->md5sum, ((hash*)t->data)->cnt); |
|---|
| | 104 | |
|---|
| | 105 | |
|---|
| | 106 | Q_RLOCK(&clusterq->lock); |
|---|
| | 107 | |
|---|
| | 108 | // connect all clusters within range |
|---|
| | 109 | for (cur_cqelem = clusterq->head; cur_cqelem; cur_cqelem = cur_cqelem->next) { |
|---|
| | 110 | for (cur_hqelem = ((cluster *)cur_cqelem->data)->hq->head; cur_hqelem; cur_hqelem = cur_hqelem->next) { |
|---|
| 166 | | // create new cluster of two outliers |
|---|
| 167 | | if ((tmp_cqelem = queue_ins(clusterq, create_cluster(((hash*)t->data), tmp_hash), clusterq_max)) != NULL) { |
|---|
| 168 | | /* cluster queue is full, last element was dropped and must be free()d */ |
|---|
| 169 | | |
|---|
| 170 | | /* first remove hashes from tries */ |
|---|
| 171 | | for (tmp_hqelem = ((cluster *)tmp_cqelem->data)->hq->head; tmp_hqelem; tmp_hqelem=tmp_hqelem->next) { |
|---|
| 172 | | |
|---|
| 173 | | pthread_rwlock_wrlock(&md5sum_trie_lock); |
|---|
| 174 | | trie_del_memstr(&md5sum_trie, (u_char *) ((hash *)tmp_hqelem->data)->md5sum, |
|---|
| 175 | | strlen(((hash *)tmp_hqelem->data)->md5sum)); |
|---|
| 176 | | pthread_rwlock_unlock(&md5sum_trie_lock); |
|---|
| 177 | | |
|---|
| 178 | | pthread_rwlock_wrlock(&spamsum_trie_lock); |
|---|
| 179 | | trie_del_memstr(&spamsum_trie, (u_char *) ((hash *)tmp_hqelem->data)->spamsum, |
|---|
| 180 | | strlen(((hash *)tmp_hqelem->data)->spamsum)); |
|---|
| 181 | | pthread_rwlock_unlock(&spamsum_trie_lock); |
|---|
| 182 | | } |
|---|
| 183 | | /* now free cluster */ |
|---|
| 184 | | cluster_free((cluster *)tmp_cqelem->data); |
|---|
| 185 | | free(tmp_cqelem); |
|---|
| 186 | | } |
|---|
| 187 | | /* set cluster element's parent pointer to cluster queue head |
|---|
| 188 | | * we need this to be able to unlink a cluster from the queue */ |
|---|
| 189 | | ((cluster *) clusterq->head->data)->parent = clusterq->head; |
|---|
| 190 | | |
|---|
| 191 | | if (verbose) printf(" New Cluster created.\n"); |
|---|
| | 134 | if ((cluster *) cur_cqelem->data != ((hash*)t->data)->cl) |
|---|
| | 135 | clusters_merge(clusterq, (cluster *) cur_cqelem->data, ((hash*)t->data)->cl); |
|---|
| | 136 | break; |
|---|
| 194 | | if (!cur_hqelem) break; |
|---|
| 195 | | } |
|---|
| 196 | | |
|---|
| 197 | | |
|---|
| 198 | | Q_ULOCK(&clusterq->lock); |
|---|
| 199 | | |
|---|
| 200 | | |
|---|
| 201 | | |
|---|
| 202 | | if (verbose > 1) printf(" Spamsum is %s (%u instances)\n", ((hash*)t->data)->spamsum, ((hash*)t->data)->cnt); |
|---|
| 203 | | |
|---|
| 204 | | if (!((hash*)t->data)->cl) { |
|---|
| 205 | | // insert outlier into queue |
|---|
| 206 | | if (outlierq->size >= outlierq_max) { |
|---|
| 207 | | tmp_hash = queue_unlink(outlierq, outlierq->tail); |
|---|
| 208 | | |
|---|
| 209 | | pthread_rwlock_wrlock(&md5sum_trie_lock); |
|---|
| 210 | | trie_del_memstr(&md5sum_trie, (u_char *) tmp_hash->md5sum, strlen(tmp_hash->md5sum)); |
|---|
| 211 | | pthread_rwlock_unlock(&md5sum_trie_lock); |
|---|
| 212 | | |
|---|
| 213 | | pthread_rwlock_wrlock(&spamsum_trie_lock); |
|---|
| 214 | | trie_del_memstr(&spamsum_trie, (u_char *) tmp_hash->spamsum, strlen(tmp_hash->spamsum)); |
|---|
| 215 | | pthread_rwlock_unlock(&spamsum_trie_lock); |
|---|
| 216 | | |
|---|
| 217 | | hash_free(tmp_hash); |
|---|
| | 139 | } |
|---|
| | 140 | } |
|---|
| | 141 | |
|---|
| | 142 | // connect all outliers within range |
|---|
| | 143 | for (cur_hqelem = outlierq->head; cur_hqelem; cur_hqelem = cur_hqelem->next) { |
|---|
| | 144 | if ((score = spamsum_match(((hash*)t->data)->spamsum, ((hash*)cur_hqelem->data)->spamsum)) >= cluster_radius) { |
|---|
| | 145 | // unlink match from outlier list |
|---|
| | 146 | |
|---|
| | 147 | tmp_hqelem = cur_hqelem; |
|---|
| | 148 | cur_hqelem = cur_hqelem->next; |
|---|
| | 149 | if ((tmp_hash = queue_unlink(outlierq, tmp_hqelem)) == NULL) { |
|---|
| | 150 | fprintf(stderr, "Error - Unable to unlink outlier from queue.\n"); |
|---|
| | 151 | exit(EXIT_FAILURE); |
|---|
| 219 | | queue_ins(outlierq, t->data, outlierq_max); |
|---|
| 220 | | |
|---|
| 221 | | if (verbose) printf(" Input added to outlier queue (maximum score: %1f, outlier queue size: %lu).\n", |
|---|
| 222 | | max_score, outlierq->size); |
|---|
| 223 | | } |
|---|
| | 153 | if (((hash*)t->data)->cl) { |
|---|
| | 154 | // add other outliers to cluster |
|---|
| | 155 | Q_ULOCK(&clusterq->lock); |
|---|
| | 156 | Q_WLOCK(&clusterq->lock); |
|---|
| | 157 | if ((qtail = add_entry_to_cluster(((hash*)t->data)->cl, tmp_hash, outlierq_max)) != NULL) { |
|---|
| | 158 | tmp_hash = qtail->data; |
|---|
| | 159 | |
|---|
| | 160 | pthread_rwlock_wrlock(&md5sum_trie_lock); |
|---|
| | 161 | trie_del_memstr(&md5sum_trie, (u_char *) tmp_hash->md5sum, strlen(tmp_hash->md5sum)); |
|---|
| | 162 | pthread_rwlock_unlock(&md5sum_trie_lock); |
|---|
| | 163 | |
|---|
| | 164 | pthread_rwlock_wrlock(&spamsum_trie_lock); |
|---|
| | 165 | trie_del_memstr(&spamsum_trie, (u_char *) tmp_hash->spamsum, strlen(tmp_hash->spamsum)); |
|---|
| | 166 | pthread_rwlock_unlock(&spamsum_trie_lock); |
|---|
| | 167 | |
|---|
| | 168 | hash_free(tmp_hash); |
|---|
| | 169 | free(qtail); |
|---|
| | 170 | } else if (verbose) printf(" Cluster has now %u elements (threshold is %lu).\n", |
|---|
| | 171 | ((hash*)t->data)->cl->hq->size, ((hash*)t->data)->cl->threshold); |
|---|
| | 172 | } else { |
|---|
| | 173 | // create new cluster of two outliers |
|---|
| | 174 | Q_ULOCK(&clusterq->lock); |
|---|
| | 175 | Q_WLOCK(&clusterq->lock); |
|---|
| | 176 | if ((tmp_cqelem = queue_ins(clusterq, create_cluster(((hash*)t->data), tmp_hash, clusterhashq_max), |
|---|
| | 177 | clusterq_max)) != NULL) { |
|---|
| | 178 | /* cluster queue is full, last element was dropped and must be free()d */ |
|---|
| | 179 | |
|---|
| | 180 | /* first remove hashes from tries */ |
|---|
| | 181 | for (tmp_hqelem = ((cluster *)tmp_cqelem->data)->hq->head; tmp_hqelem; tmp_hqelem=tmp_hqelem->next) { |
|---|
| | 182 | |
|---|
| | 183 | pthread_rwlock_wrlock(&md5sum_trie_lock); |
|---|
| | 184 | trie_del_memstr(&md5sum_trie, (u_char *) ((hash *)tmp_hqelem->data)->md5sum, |
|---|
| | 185 | strlen(((hash *)tmp_hqelem->data)->md5sum)); |
|---|
| | 186 | pthread_rwlock_unlock(&md5sum_trie_lock); |
|---|
| | 187 | |
|---|
| | 188 | pthread_rwlock_wrlock(&spamsum_trie_lock); |
|---|
| | 189 | trie_del_memstr(&spamsum_trie, (u_char *) ((hash *)tmp_hqelem->data)->spamsum, |
|---|
| | 190 | strlen(((hash *)tmp_hqelem->data)->spamsum)); |
|---|
| | 191 | pthread_rwlock_unlock(&spamsum_trie_lock); |
|---|
| | 192 | } |
|---|
| | 193 | /* now free cluster */ |
|---|
| | 194 | cluster_free((cluster *)tmp_cqelem->data); |
|---|
| | 195 | free(tmp_cqelem); |
|---|
| | 196 | } |
|---|
| | 197 | /* set cluster element's parent pointer to cluster queue head |
|---|
| | 198 | * we need this to be able to unlink a cluster from the queue */ |
|---|
| | 199 | ((cluster *) clusterq->head->data)->parent = clusterq->head; |
|---|
| | 200 | |
|---|
| | 201 | if (verbose) printf(" New Cluster created.\n"); |
|---|
| | 202 | } |
|---|
| | 203 | Q_ULOCK(&clusterq->lock); |
|---|
| | 204 | Q_RLOCK(&clusterq->lock); |
|---|
| | 205 | } else if (score > max_score) max_score = score; |
|---|
| | 206 | if (!cur_hqelem) break; |
|---|
| | 207 | } |
|---|
| | 208 | |
|---|
| | 209 | Q_ULOCK(&clusterq->lock); |
|---|
| | 210 | |
|---|
| | 211 | |
|---|
| | 212 | if (verbose > 1) printf(" Spamsum is %s (%u instances)\n", ((hash*)t->data)->spamsum, ((hash*)t->data)->cnt); |
|---|
| | 213 | |
|---|
| | 214 | if (!((hash*)t->data)->cl) { |
|---|
| | 215 | // insert outlier into queue |
|---|
| | 216 | if (outlierq->size >= outlierq_max) { |
|---|
| | 217 | tmp_hash = queue_unlink(outlierq, outlierq->tail); |
|---|
| | 218 | |
|---|
| | 219 | pthread_rwlock_wrlock(&md5sum_trie_lock); |
|---|
| | 220 | trie_del_memstr(&md5sum_trie, (u_char *) tmp_hash->md5sum, strlen(tmp_hash->md5sum)); |
|---|
| | 221 | pthread_rwlock_unlock(&md5sum_trie_lock); |
|---|
| | 222 | |
|---|
| | 223 | pthread_rwlock_wrlock(&spamsum_trie_lock); |
|---|
| | 224 | trie_del_memstr(&spamsum_trie, (u_char *) tmp_hash->spamsum, strlen(tmp_hash->spamsum)); |
|---|
| | 225 | pthread_rwlock_unlock(&spamsum_trie_lock); |
|---|
| | 226 | |
|---|
| | 227 | hash_free(tmp_hash); |
|---|
| | 228 | free(qtail); |
|---|
| | 229 | } |
|---|
| | 230 | queue_ins(outlierq, t->data, outlierq_max); |
|---|
| | 231 | |
|---|
| | 232 | if (verbose) printf(" Input added to outlier queue (maximum score: %.1f, outlier queue size: %lu).\n", |
|---|
| | 233 | max_score, (long unsigned) outlierq->size); |
|---|