🚀 go-pugleaf

    Here is a little program I wrote to collect statistics from the
    .overview files in a news hierarchy.  I am providing this program
    as-is.  Please, no email regarding improvements... it's in the
    public domain, do whatever you like with it.

    You WILL have to modify it according to the way your news hierarchy 
    is organized.  You must run .overview files for this program to
    work, as that is where it extracts the information from.

    The program collects statistics based on the hostname
    part of the Message-ID field and the From: field.  Results
    should not be taken verbatim or acted upon without taking 
    into account possible counterfeiting.

    I will be posting statistics calculated from our full news spool 
    on a weekly basis to ba.news.stats.

    I am hoping that ISP's that have so far taken no interest in
    curbing posting volume abuse by their users will find it in their
    hearts to address the problem based on information they glean
    from this program, not to mention peer pressure from other entities.

    The second-easiest way to prevent bashing is to turn off specific
    newsgroups.

    The EASIEST way to prevent bashing is to simply alias-out
    the host the news was posted from in the newsfeeds file. 
    Unfortunately, this can result in sites getting wiped off the
    face of the news map.

					-Matt
					BEST Internet Communications, Inc.

/*
 * SITECOLL.C
 *
 * Collect statistics on the source of news
 *
 * SITECOLL				run through active file and scan
 *					all .overview files for active groups
 *
 * SITECOLL <overviewfiles....>		scan only specific .overview files
 *
 * TO MAKE THIS WORK ON YOUR MACHINE:
 *
 *	Change the two references to /home/news in the code to point to
 *	the right place.  If your .overview files do not reside in the
 *	same directories as the articles, you will have to fix that too.
 *
 *	The format of your .overview file may not be compatible, you may
 *	have to change the strok() sequencing.  Our overview files are
 *	arranged as follows:
 *
 *		Subject: 
 *		From:		<-- extracted by program
 *		Date:
 *		Message-ID:	<-- extracted by program
 *		References:
 *		Bytes:		<-- extracted by program
 *		Lines:
 *		Xref:full
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

typedef struct Hash {
    struct Hash *h_Next;
    char *h_Key;
    long h_Size;
    long h_Hv;
} Hash;

#define HSIZE	32768
#define HMASK	(HSIZE-1)

int MsgHashCount;
int FromHashCount;
Hash *MsgHash[HSIZE];
Hash *FromHash[HSIZE];

void AddNode(Hash *table[HSIZE], int *cnt, const char *key, long size);
long hashstr(const char *str);
void ScanOverview(const char *path);
Hash **SortHash(Hash *table[HSIZE], int count);

int
main(int ac, char **av)
{
    char buf[1024];
    char path[1024];

    if (ac == 1) {
	FILE *fi;
	if ((fi = fopen("/home/news/active", "r")) != NULL) {
	    while (fgets(buf, sizeof(buf), fi) != NULL) {
		char *ptr;

		if ((ptr = strchr(buf, ' ')) != NULL) {
		    *ptr = 0;
		    ptr = buf;

		    while ((ptr = strchr(ptr, '.')) != NULL)
			*ptr = '/';
		    sprintf(path, "/home/news/spool/news/%s/.overview", buf);
		    fprintf(stderr, "%s\n", path);
		    ScanOverview(path);
		}
	    }
	    fclose(fi);
	}
    }
    {
	int i;

	for (i = 1; i < ac; ++i)
	    ScanOverview(av[i]);
    }
    {
	Hash **msgs;
	Hash **froms;
	int i;

	msgs = SortHash(MsgHash, MsgHashCount);
	froms = SortHash(FromHash, FromHashCount);
	for (i = 0; i < MsgHashCount; ++i)
	    printf("MSGID %d\t%s\n", msgs[i]->h_Size, msgs[i]->h_Key);
	for (i = 0; i < FromHashCount; ++i)
	    printf("FROMF %d\t%s\n", froms[i]->h_Size, froms[i]->h_Key);
    }
}

void
ScanOverview(const char *path)
{
    FILE *fi;
    char buf[8192];

    if ((fi = fopen(path, "r")) == NULL)
	return;
    while (fgets(buf, sizeof(buf), fi) != NULL) {
	char *fromFld;
	char *msgFld;
	char *sizeFld;

	if (strtok(buf, "\t\n\r") &&
	    strtok(NULL, "\t\n\r") &&
	    (fromFld = strtok(NULL, "\t\n\r")) != NULL &&
	    strtok(NULL, "\t\n\r") &&
	    (msgFld = strtok(NULL, "\t\n\r")) != NULL &&
	    strtok(NULL, "\t\n\r") &&
	    (sizeFld = strtok(NULL, "\t\n\r")) != NULL
	) {
	    if ((msgFld = strchr(msgFld, '@')) != NULL) {
		AddNode(MsgHash, &MsgHashCount, msgFld, strtol(sizeFld, NULL, 0));
		AddNode(FromHash, &FromHashCount, fromFld, strtol(sizeFld, NULL, 0));
	    }
	}
    }
    fclose(fi);
}

void
AddNode(Hash *table[HSIZE], int *cnt, const char *key, long size)
{
    Hash **phash, *hash;
    long hv = hashstr(key);

    for (phash = &table[hv & HMASK];
	(hash = *phash) != NULL;
	phash = &hash->h_Next
    ) {
	if (hash->h_Hv == hv && strcmp(hash->h_Key, key) == 0)
	    break;
    }
    if (hash == NULL) {
	hash = malloc(sizeof(Hash) + strlen(key) + 1);
	hash->h_Next = *phash;
	hash->h_Key = (char *)(hash + 1);
	hash->h_Size = 0;
	hash->h_Hv = hv;
	*phash = hash;
	strcpy(hash->h_Key, key);
	++*cnt;
    }
    hash->h_Size += size;
}

int 
compar(const void *s1, const void *s2)
{
    Hash *h1 = *(Hash **)s1;
    Hash *h2 = *(Hash **)s2;
    return(h2->h_Size - h1->h_Size);
}

Hash **
SortHash(Hash *table[HSIZE], int count)
{
    Hash *hash;
    Hash **ary;
    int i;
    int j;

    ary = malloc(sizeof(Hash *) * (count + 1));
    for (i = j = 0; i < HSIZE; ++i) {
	for (hash = table[i]; hash; hash = hash->h_Next) {
	    ary[j++] = hash;
	}
    }
    qsort(ary, count, sizeof(Hash *), compar);
    return(ary);
}

long
hashstr(const char *str)
{
    long hv = 0xAF56B000;

    while (*str) {
	hv = (hv << 5) | *str | (hv >> 23);
	++str;
    }
    return(hv);
}

-- 
    Matthew Dillon   Engineering, BEST Internet Communications, Inc.
		    <dillon@best.com>, <dillon@apollo.best.com>
    [always include a portion of the original email in any response!]