Navigation

← Previous Changeset
Next Changeset →

Changeset 1418

Timestamp:

10/14/07 13:53:29 (1 year ago)

Author:

till

Message:

nebula
- throughput profiling
- clean configure script

Files:

nebula/trunk/ChangeLog (modified) (1 diff)
nebula/trunk/configure.in (modified) (1 diff)
nebula/trunk/src/Makefile.am (modified) (1 diff)
nebula/trunk/src/nebula.c (modified) (8 diffs)
nebula/trunk/src/nebula.h (modified) (2 diffs)
nebula/trunk/src/signals.c (modified) (4 diffs)
nebula/trunk/src/signals.h (modified) (1 diff)
nebula/trunk/src/util.c (modified) (4 diffs)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

nebula/trunk/ChangeLog

r1416	r1418
	1	- throughput profiling
	2	- clean configure script
1	3	- process input from directory chronologically when -t is given
2	4	- fixed sized queues for outlier list and cluster elements; size can be set via command line options

nebula/trunk/configure.in

r1373	r1418
1	1	# $Id$
2	2	AC_PREREQ(02.50)
3		AC_INIT([~~cluster~~], [0.0.1], [tillmann.werner@gmx.de])
	3	AC_INIT([nebula], [0.0.1], [tillmann.werner@gmx.de])
4	4	AM_CONFIG_HEADER(config.h)
5		AM_INIT_AUTOMAKE(~~cluster~~,0.0.1)
	5	AM_INIT_AUTOMAKE(nebula,0.0.1)
6	6
7		~~# Since we get -O2 from configure defaults, which doesn't work in 64bit~~
8		~~# mode, let's make some changes here before calling _CC macros.~~
9		~~AC_ARG_ENABLE(64bit-gcc,~~
10		~~[ --enable-64bit-gcc try to compile 64bit (only tested on Sparc Solaris 9).],~~
11		~~[ CFLAGS="-O0 -g" CC="gcc -m64"; export CFLAGS CC ],)~~
12		~~# Disable annoying practice of recursively re-running the autotools~~
13		~~AM_MAINTAINER_MODE~~
14		~~AM_PROG_CC_STDC~~
15	7	AC_PROG_CC
16		~~#AC_PROG_LD(gnu-ld)~~
17		~~#AC_PROG_LIBTOOL~~
18		~~if test -n "$GCC"; then~~
19		~~CFLAGS="$CFLAGS -Wall"~~
20		fi
21	8
22	9	AC_ARG_ENABLE(debug,
23	10	[ --enable-debug enable debugging options (bugreports and developers only)],
24		[ if test -n "$GCC"; then
25		CFLAGS="-O0 -DDEBUG -g"
26		else
27		CFLAGS="$CFLAGS -DDEBUG"
28		fi
29		~~enable_debug="X"~~
30		], enable_debug=" ")
	11	[ if test -n "$GCC"; then
	12	CFLAGS="-O0 -DDEBUG -g"
	13	else
	14	CFLAGS="$CFLAGS -DDEBUG"
	15	fi
	16	enable_debug="X"], enable_debug=" ")
	17
31	18
32	19	AC_ARG_ENABLE(profile,
33	20	[ --enable-profile enable profiling options (developers only)],
34		[ if test -n "$GCC"; then
35		CFLAGS="$CFLAGS -DPROFILE -pg"
36		else
37		CFLAGS="$CFLAGS -DPROFILE"
38		fi
39		enable_profile="X"
40		], enable_profile=" ")
41
42		#AC_CANONICAL_HOST
43		linux=no
44		sunos4=no
45
46		AC_C_BIGENDIAN
47		SHELL="/bin/sh"
48
49		case "$host" in
50		-openbsd2.6\|-openbsd2.5\|-openbsd2.4\|-openbsd2.3*)
51		AC_DEFINE(OPENBSD,,[Define if OPENBSD])
52		AC_DEFINE(BROKEN_SIOCGIFMTU,,[Define if BROKEN_SIOCGIFMTU])
53
54		;;
55		-openbsd)
56		AC_DEFINE(OPENBSD)
57
58		;;
59		-sgi-irix5)
60		AC_DEFINE(IRIX,,[Define if IRIX])
61		no_libsocket=yes
62		no_libnsl=yes
63		if test -z "$GCC"; then
64		sgi_cc=yes
65		fi
66		LDFLAGS=${LDFLAGS} -L/usr/local/lib
67		extra_incl=-I/usr/local/include
68		;;
69		-sgi-irix6)
70		AC_DEFINE(IRIX)
71		no_libsocket=yes
72		no_libnsl=yes
73		if test -z "$GCC"; then
74		sgi_cc=yes
75		fi
76		LDFLAGS=${LDFLAGS} -L/usr/local/lib
77		extra_incl=-I/usr/local/include
78		;;
79		-solaris)
80		AC_DEFINE(SOLARIS,,[Define if SOLARIS])
81		CPPFLAGS="${CPPFLAGS} -DBSD_COMP -D_REENTRANT"
82		;;
83		-sunos)
84		AC_DEFINE(SUNOS,,[Define if SUNOS])
85		sunos4=yes
86		;;
87		-linux)
88		linux=yes
89		AC_DEFINE(LINUX,,[Define if LINUX])
90		# libpcap doesn't even LOOK at the timeout you give it under Linux
91		AC_DEFINE(PCAP_TIMEOUT_IGNORED,,[Define if PCAP_TIMEOUT_IGNORED])
92		AC_SUBST(extra_incl)
93		extra_incl=-I/usr/include/pcap
94		;;
95		-hpux10)
96		AC_DEFINE(HPUX,,[Define if HPUX])
97		AC_DEFINE(WORDS_BIGENDIAN)
98		AC_SUBST(extra_incl)
99		extra_incl=-I/usr/local/include
100		;;
101
102		-freebsd)
103		AC_DEFINE(FREEBSD,,[Define if FREEBSD])
104		CFLAGS="$CFLAGS -fPIC -DPIC"
105
106		;;
107		-bsdi)
108		AC_DEFINE(BSDI,,[Define if BSDI])
109		;;
110		-aix)
111		AC_DEFINE(AIX,,[Define if AIX])
112		broken_types=yes
113		;;
114		-osf4)
115		AC_DEFINE(OSF1,,[Define if OSF1])
116		tru64_types=yes
117		;;
118		-osf5.1)
119		AC_DEFINE(OSF1)
120		;;
121		-tru64)
122		AC_DEFINE(OSF1)
123		tru64_types=yes
124		;;
125		# it is actually <platform>-apple-darwin1.2 or <platform>-apple-rhapsody5.x but lets stick with this for the moment
126		-apple)
127		AC_DEFINE(MACOS,,[Define if MACOS])
128		AC_DEFINE(BROKEN_SIOCGIFMTU)
129		LDFLAGS="${LDFLAGS} -L/sw/lib"
130		extra_incl=-I/sw/include
131		esac
132
133		# any sparc platform has to have this one defined.
134		AC_MSG_CHECKING(for sparc alignment)
135		if eval "echo $host_cpu\|grep -i sparc >/dev/null"; then
136		AC_DEFINE(WORDS_MUSTALIGN,,[Define if WORDS_MUSTALIGN])
137		AC_MSG_RESULT(yes)
138		else
139		AC_MSG_RESULT(no)
140		fi
141
142
143		LIBS="${LIBS}"
144
145		AC_TRY_COMPILE([
146		#include <stdio.h>
147		],[char *foo; foo = sys_errlist[0];],
148		AC_DEFINE(ERRLIST_PREDEFINED,,[Define if ERRLIST_PREDEFINED]))
149
150		AC_MSG_CHECKING(for __FUNCTION__)
151		AC_TRY_COMPILE([
152		#include <stdio.h>
153		],[printf ("%s", __FUNCTION__);],
154		sn_cv_have___FUNCTION__=yes, sn_cv__have___FUNCTION__=no)
155		if test "x$sn_cv_have___FUNCTION__" = "xyes"; then
156		AC_MSG_RESULT(yes)
157		AC_DEFINE(HAVE___FUNCTION__, 1,
158		[Define if the compiler understands __FUNCTION__.])
159		else
160		AC_MSG_RESULT(no)
161		AC_MSG_CHECKING(for __func__)
162		AC_TRY_COMPILE([
163		#include <stdio.h>
164		],[printf ("%s", __func__);],
165		sn_cv_have___func__=yes, sn_cv__have___func__=no)
166		if test "x$sn_cv_have___func__" = "xyes"; then
167		AC_MSG_RESULT(yes)
168		AC_DEFINE(HAVE___func__, 1,
169		[Define if the compiler understands __func__.])
170		AC_DEFINE(__FUNCTION__, __func__, [Define __FUNCTION__ as required.])
171		else
172		AC_MSG_RESULT(no)
173		AC_DEFINE(__FUNCTION__, "mystery function")
174		fi
175		fi
	21	[ if test -n "$GCC"; then
	22	CFLAGS="$CFLAGS -DPROFILE -pg"
	23	else
	24	CFLAGS="$CFLAGS -DPROFILE"
	25	fi
	26	enable_profile="X"], enable_profile=" ")
176	27
177	28
178	29	# Check for electric fence malloc debugger
179		AC_ARG_WITH(efence, [ --with-efence link with electric fence ])
180		if test "$with_efence" = "yes"
181		then
182		with_efence="X"
183		AC_CHECK_LIB(efence, EF_ALIGNMENT, LIBS="${LIBS} -lefence", AC_MSG_ERROR(libefence not found))
184		else
185		with_efence=" "
186		fi
187
188
189		# let's make some fixes..
190
191		CFLAGS=`echo $CFLAGS \|sed -e 's/-I\/usr\/include //g'`
192		CPPFLAGS=`echo $CPPFLAGS \| sed -e 's/-I\/usr\/include //g'`
193
194		INCLUDES='-I$(top_srcdir) -I$(top_srcdir)/src'
195
196		AC_SUBST(INCLUDES)
	30	AC_ARG_ENABLE(efence,
	31	[ --enable-efence link with electric fence], enable_efence="X", enable_efence=" ")
	32	if test "$enable_efence" = "X"; then
	33	AC_CHECK_LIB(efence, EF_ALIGNMENT, LIBS="${LIBS} -lefence", AC_MSG_ERROR(libefence not found))
	34	fi
197	35
198	36

nebula/trunk/src/Makefile.am

r1414 r1418

1 1 AM_CFLAGS=-Wall -Werror
2 2
3 LIBS=-lm
3 LIBS += -lm
4 4
5 5 bin_PROGRAMS = nebula

nebula/trunk/src/nebula.c

r1416	r1418
32	32	#include <unistd.h>
33	33
	34	#if HAVE_CONFIG_H
	35	//#include <config.h>
	36	#endif
	37
34	38	#include "cluster.h"
35	39	#include "md5.h"
…	…
40	44	#include "util.h"
41	45
	46	#ifndef VERSION
	47	#define VERSION "(unknown version)"
	48	#endif
	49
42	50
43	51	void usage(const char* progname, const int exit_val) {
44		printf("Usage: %s [-l] [-v] [-r cluster radius] [-d directory] [file ...]\n", progname);
	52	#ifdef PROFILE
	53	printf("Usage: %s [-ltpv] [-c maximum cluster size] [-r cluster radius] [-q maximum outlier queue size] [-d directory] [file ...]\n", progname);
	54	#else
	55	printf("Usage: %s [-ltpv] [-a profiling interval] [-c maximum cluster size] [-r cluster radius] [-q maximum outlier queue size] [-d directory] [file ...]\n", progname);
	56	#endif
45	57
46	58	exit(exit_val);
…	…
48	60
49	61	int main(int argc, char *argv[]) {
50		int i, qsize~~, show_progress~~;
	62	int i, qsize;
51	63	u_char time_sort, content, tmpbuf;
52	64	char option, *dirname;
…	…
72	84	qsize = 0;
73	85	total_files = 0;
	86	#ifdef PROFILE
	87	alarm_time = 5;
	88	#endif
74	89
75	90	/* default values for parameters */
…	…
85	100	memset(&spamsum_trie, 0, sizeof(trie_node));
86	101
	102	printf("\n Nebula %s Copyright (C) 2007 Tillmann Werner <tillmann.werner@gmx.de>\n\n", VERSION);
	103
87	104	// process args
	105	#ifdef PROFILE
	106	while((option = getopt(argc, argv, "c:q:a:tplvd:r:h?")) > 0) {
	107	#else
88	108	while((option = getopt(argc, argv, "c:q:tplvd:r:h?")) > 0) {
	109	#endif
89	110	switch(option) {
	111	#ifdef PROFILE
	112	case 'a':
	113	alarm_time = atoi(optarg);
	114	if (alarm_time < 1) {
	115	fprintf(stderr, "Error - Profile time interval must be at least one second.\n");
	116	exit(EXIT_FAILURE);
	117	}
	118	break;
	119	#endif
90	120	case 'c':
91	121	clusterq_max = atoi(optarg);
…	…
137	167	set_signal_handlers();
138	168
139		/* initialize outlier queue */
140		outlierq = hashq_new();
141
	169
	170	// if a directory is given, scan (and sort) its content
142	171	if (dirname) {
143		printf("scanning directory... ");
	172	printf("Scanning directory... ");
144	173	fflush(stdout);
145	174	if (time_sort) {
…	…
162	191	}
163	192
	193	// initialize outlier queue
	194	outlierq = hashq_new();
	195
	196
	197	#ifdef PROFILE
	198	alarm(alarm_time);
	199	printf("Profiling enabled, printing statistics every %d seconds.\n", alarm_time);
	200	files_in_interval = 0;
	201	checkpoint = 0;
	202	#endif
164	203
165	204	printf("processing %u files.\n", (unsigned int) total_files);
…	…
169	208	}
170	209
	210
171	211	for (n=0; n<total_files; n++) {
172	212	bstr = bstr_map(namelist[n]->d_name);
173	213	num_of_files++;
	214	#ifdef PROFILE
	215	files_in_interval++;
	216	#endif
174	217
175	218	if (verbose) printf(" processing file %s.\n", namelist[n]->d_name);
176	219	else if (show_progress) {
177		printf("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b");
	220	printf("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b");
178	221	if (num_of_files < total_files)
179		printf("%05.2f%% (%04d clusters)", num_of_files/total_files*100, num_of_clusters);
	222	printf("files processed: %05.2f%% (%04d clusters)", num_of_files/total_files*100, num_of_clusters);
180	223	else
181		printf("100.00%% (%04d clusters)\n", num_of_clusters);
	224	printf("files processed: 100.00%% (%04d clusters)\n", num_of_clusters);
182	225	fflush(stdout);
183	226	}

nebula/trunk/src/nebula.h

r1416	r1418
31	31	#include "hashq.h"
32	32
33		u_char verbose, list_files;
	33	u_char verbose, list_files, show_progress;
34	34	int clusterq_max, outlierq_max;
35	35	u_int16_t num_of_clusters;
…	…
40	40	hashq *outlierq;
41	41
	42	#ifdef PROFILE
	43	int alarm_time; // number of seconds for profile output interval
	44	float files_in_interval;
	45	u_int32_t checkpoint;
42	46	#endif
	47
	48	#endif

nebula/trunk/src/signals.c

r1414	r1418
23	23	#include <stdlib.h>
24	24	#include <string.h>
	25	#include <unistd.h>
25	26
26	27	#include "nebula.h"
…	…
29	30
30	31
31		void handle_signal(int sig) {
	32	void handle_term_signal(int sig) {
32	33	printf("Premature termination forced (signal %d caught).\n", sig);
	34
33	35	cleanup();
34	36	exit(EXIT_SUCCESS);
…	…
37	39
38	40
	41	#ifdef PROFILE
	42	void handle_alarm_signal(int sig) {
	43	if (show_progress) printf("\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\n");
	44	printf("%u: %u files form %u clusters (%.0f files per second).\n",
	45	checkpoint, (unsigned int) num_of_files, num_of_clusters, files_in_interval/alarm_time);
	46
	47	checkpoint++;
	48	files_in_interval = 0;
	49	alarm(alarm_time);
	50
	51	return;
	52	}
	53	#endif
	54
	55
39	56	void set_signal_handlers(void) {
40	57	u_char i;
41	58	struct sigaction s_action;
42		static int sigs[] = {
	59	static int termsigs[] = {
43	60	#ifdef HAVE_SIGBUS
44	61	SIGBUS,
…	…
52	69	};
53	70
54		/* install signal handlers */
	71	// install handler for signals indicating errors
55	72	memset(&s_action, 0, sizeof(struct sigaction));
56		s_action.sa_handler = handle_signal;
	73	s_action.sa_handler = handle_term_signal;
57	74	#ifdef SA_RESTART
58	75	s_action.sa_flags \|= SA_RESTART;
59	76	#endif
60		for (i = 0; i < sizeof(~~sigs)/sizeof(~~sigs[0]); i++) {
61		if (sigaction(sigs[i], &s_action, NULL) == -1) {
62		fprintf(stdout, " Error - Unable to install handler for signal %d: %m.\n", sigs[i]);
	77	for (i = 0; i < sizeof(termsigs)/sizeof(termsigs[0]); i++) {
	78	if (sigaction(termsigs[i], &s_action, NULL) == -1) {
	79	fprintf(stdout, " Error - Unable to install handler for signal %d: %m.\n", termsigs[i]);
63	80	exit(EXIT_FAILURE);
64	81	}
65	82	}
	83
	84	#ifdef PROFILE
	85	// install handler for SIGALARM
	86	memset(&s_action, 0, sizeof(struct sigaction));
	87	s_action.sa_handler = handle_alarm_signal;
	88	#ifdef SA_RESTART
	89	s_action.sa_flags \|= SA_RESTART;
	90	#endif
	91	if (sigaction(SIGALRM, &s_action, NULL) == -1) {
	92	fprintf(stdout, " Error - Unable to install handler for signal %d: %m.\n", SIGALRM);
	93	exit(EXIT_FAILURE);
	94	}
	95	#endif
	96
66	97	return;
67	98	}

nebula/trunk/src/signals.h

r1376	r1418
27	27
28	28
29		void handle_signal(int sig);
	29	void handle_term_signal(int sig);
	30	#ifdef PROFILE
	31	void handle_alarm_signal(int sig);
	32	#endif
30	33	void set_signal_handlers(void);
31	34

nebula/trunk/src/util.c

r1415	r1418
21	21	#include <dirent.h>
22	22	#include <fcntl.h>
	23	#include <signal.h>
23	24	#include <stdlib.h>
24	25	#include <stdio.h>
…	…
31	32	#include "hashq.h"
32	33	#include "nebula.h"
	34	#include "signals.h"
33	35	#include "trie.h"
34	36	#include "util.h"
	37
35	38
36	39	bstring bstr_map(const char* filename) {
…	…
103	106	}
104	107
105		if (sta.st_ctime < stb.st_ctime) return(1);
106		if (sta.st_ctime > stb.st_ctime) return(-1);
	108	if (sta.st_ctime < stb.st_ctime) return(-1);
	109	if (sta.st_ctime > stb.st_ctime) return(1);
107	110	return(0);
108	111	}
…	…
110	113
111	114	void cleanup(void) {
	115	struct sigaction s_action;
	116
	117	#ifdef PROFILE
	118	// reset handler for SIGALRM
	119	memset(&s_action, 0, sizeof(struct sigaction));
	120	s_action.sa_handler = NULL;
	121	#ifdef SA_RESTART
	122	s_action.sa_flags \|= SA_RESTART;
	123	#endif
	124	if (sigaction(SIGALRM, &s_action, NULL) == -1) {
	125	fprintf(stdout, " Error - Unable to reset handler for signal %d: %m.\n", SIGALRM);
	126	exit(EXIT_FAILURE);
	127	}
	128	#endif
	129
112	130	// free data structures
113	131	printf("%u files form %u clustes.\n-----------------------\n", (unsigned int) num_of_files, num_of_clusters);

Download in other formats: