529 lines
12 KiB
HolyC
Executable File
529 lines
12 KiB
HolyC
Executable File
#define DOWNLOAD_FILE1 "TOS_Distro.ISO"
|
||
#define DOWNLOAD_FILE1_SIZE 16000000
|
||
|
||
#define HOURS_MAX (24*3)
|
||
|
||
class LogStruct
|
||
{
|
||
LogStruct *next,*last;
|
||
LogStruct *ip_num_left,*ip_num_right;
|
||
U32 ip_num,code;
|
||
I64 size;
|
||
U8 *file,*link;
|
||
CDate datetime;
|
||
};
|
||
|
||
class LinkStruct
|
||
{
|
||
LinkStruct *left,*right;
|
||
U8 *link,*file;
|
||
I64 cnt;
|
||
};
|
||
|
||
class BlockedStruct
|
||
{
|
||
BlockedStruct *next,*last;
|
||
U32 ip_num;
|
||
};
|
||
|
||
U0 LogStructDel(LogStruct *tmplg)
|
||
{
|
||
Free(tmplg->file);
|
||
Free(tmplg->link);
|
||
Free(tmplg);
|
||
}
|
||
|
||
U0 PrsSingleLogFile(LogStruct *head,U8 *name,CDate *_dstart,CDate *_dend)
|
||
{
|
||
CDoc *doc=DocRead(name,
|
||
DOCF_PLAIN_TEXT_TABS|DOCF_DBL_DOLLARS|DOCF_NO_CURSOR);
|
||
CDocEntry *doc_e=doc->head.next;
|
||
U8 *src,*src2,*mon_lst=Define("ST_MONTHS");
|
||
LogStruct *tmplg;
|
||
CDateStruct ds;
|
||
I64 i;
|
||
"%$$Q\n",name;
|
||
while (doc_e!=doc) {
|
||
if (doc_e->type_u8==DOCT_TEXT) {
|
||
tmplg=CAlloc(sizeof(LogStruct));
|
||
try {
|
||
src=doc_e->tag;
|
||
tmplg->ip_num.u8[3]=Str2I64(src,10,&src);
|
||
if (*src++!='.') throw;
|
||
tmplg->ip_num.u8[2]=Str2I64(src,10,&src);
|
||
if (*src++!='.') throw;
|
||
tmplg->ip_num.u8[1]=Str2I64(src,10,&src);
|
||
if (*src++!='.') throw;
|
||
tmplg->ip_num.u8[0]=Str2I64(src,10,&src);
|
||
|
||
do if (!*src) throw;
|
||
while (*src++!='[');
|
||
MemSet(&ds,0,sizeof(CDateStruct));
|
||
ds.day_of_mon=Str2I64(src,10,&src);
|
||
if (*src++!='/') throw;
|
||
src2=src;
|
||
do if (!*src2) throw;
|
||
while (*src2++!='/');
|
||
* --src2=0;
|
||
ds.mon=1+LstMatch(src,mon_lst,LMF_IGNORE_CASE);
|
||
src=++src2;
|
||
ds.year=Str2I64(src,10,&src);
|
||
if (*src++!=':') throw;
|
||
ds.hour=Str2I64(src,10,&src);
|
||
if (*src++!=':') throw;
|
||
ds.min=Str2I64(src,10,&src);
|
||
if (*src++!=':') throw;
|
||
ds.sec=Str2I64(src,10,&src);
|
||
tmplg->datetime=Struct2Date(&ds);
|
||
if (*src++!=CH_SPACE) throw;
|
||
i=Str2I64(src,10,&src);
|
||
tmplg->datetime-=(i/100+i%100/60.0)*CDATE_FREQ*60*60;
|
||
if (!(Str2Date("1/1/2017")<=tmplg->datetime<Str2Date("1/1/2050")))
|
||
throw;
|
||
if (tmplg->datetime<*_dstart) *_dstart=tmplg->datetime;
|
||
if (tmplg->datetime>*_dend) *_dend =tmplg->datetime;
|
||
|
||
do if (!*src) throw;
|
||
while (*src++!=']');
|
||
|
||
if (*src++!=CH_SPACE) throw;
|
||
if (*src++!='\"') throw;
|
||
if (!StrNCmp(src,"GET ",4)) {
|
||
src2=src+=4;
|
||
do if (!*src2) throw;
|
||
while (*src2++!=CH_SPACE);
|
||
* --src2=0;
|
||
tmplg->file=StrNew(src);
|
||
src=++src2;
|
||
|
||
do if (!*src) throw;
|
||
while (*src++!='\"');
|
||
|
||
tmplg->code=Str2I64(src,10,&src);
|
||
if (*src++!=CH_SPACE) throw;
|
||
tmplg->size=Str2I64(src,10,&src);
|
||
if (*src++!=CH_SPACE) throw;
|
||
|
||
if (*src++!='\"') throw;
|
||
src2=src;
|
||
do if (!*src2) throw;
|
||
while (*src2++!='\"');
|
||
* --src2=0;
|
||
tmplg->link=StrNew(src);
|
||
src=++src2;
|
||
|
||
QueIns(tmplg,head->last);
|
||
} else if (!StrNCmp(src,"HEAD ",5)) {
|
||
LogStructDel(tmplg);
|
||
} else
|
||
throw;
|
||
} catch {
|
||
Fs->catch_except=TRUE;
|
||
"%$$Q\n",doc_e->tag;
|
||
LogStructDel(tmplg);
|
||
}
|
||
}
|
||
doc_e=doc_e->next;
|
||
}
|
||
DocDel(doc);
|
||
}
|
||
|
||
LogStruct *PrsLogFiles(U8 *files_find_mask,CDate *_dstart,CDate *_dend)
|
||
{
|
||
LogStruct *head=CAlloc(sizeof(LogStruct));
|
||
CDirEntry *tmpde=FilesFind(files_find_mask),*tmpde1=tmpde;
|
||
QueInit(head);
|
||
while (tmpde) {
|
||
PrsSingleLogFile(head,tmpde->full_name,_dstart,_dend);
|
||
tmpde=tmpde->next;
|
||
}
|
||
DirTreeDel(tmpde1);
|
||
return head;
|
||
}
|
||
|
||
U0 LogLstDel(LogStruct *head)
|
||
{
|
||
LogStruct *tmplg=head->next,*tmplg1;
|
||
while (tmplg!=head) {
|
||
tmplg1=tmplg->next;
|
||
LogStructDel(tmplg);
|
||
tmplg=tmplg1;
|
||
}
|
||
}
|
||
|
||
U0 BlockedStructAdd(BlockedStruct *head,U32 ip_num)
|
||
{
|
||
BlockedStruct *tmpb=CAlloc(sizeof(BlockedStruct));
|
||
tmpb->ip_num=ip_num;
|
||
QueIns(tmpb,head->last);
|
||
}
|
||
|
||
Bool IsBlocked(BlockedStruct *head,U32 ip_num)
|
||
{
|
||
BlockedStruct *tmpb=head->next;
|
||
while (tmpb!=head) {
|
||
if (tmpb->ip_num==ip_num)
|
||
return TRUE;
|
||
tmpb=tmpb->next;
|
||
}
|
||
return FALSE;
|
||
}
|
||
|
||
U0 BlockIPNuip(LogStruct *head)
|
||
{
|
||
BlockedStruct blocked_head;
|
||
LogStruct *tmplg=head->next,*tmplg1;
|
||
|
||
QueInit(&blocked_head);
|
||
|
||
BlockedStructAdd(&blocked_head,68<<24+227<<16+61<<8+6);
|
||
|
||
//pass 1: collect robot lst
|
||
while (tmplg!=head) {
|
||
if (StrIMatch("ROBOT",tmplg->file) &&
|
||
!IsBlocked(&blocked_head,tmplg->ip_num))
|
||
BlockedStructAdd(&blocked_head,tmplg->ip_num);
|
||
tmplg=tmplg->next;
|
||
}
|
||
|
||
//pass 2: removed blocked ip_nuip
|
||
tmplg=head->next;
|
||
while (tmplg!=head) {
|
||
tmplg1=tmplg->next;
|
||
if (IsBlocked(&blocked_head,tmplg->ip_num)) {
|
||
QueRem(tmplg);
|
||
LogStructDel(tmplg);
|
||
}
|
||
tmplg=tmplg1;
|
||
}
|
||
|
||
QueDel(&blocked_head);
|
||
}
|
||
|
||
Bool IsDownLoad(LogStruct *tmplg)
|
||
{
|
||
if (StrMatch(DOWNLOAD_FILE1,tmplg->file)&&tmplg->size>=
|
||
DOWNLOAD_FILE1_SIZE)
|
||
return TRUE;
|
||
else
|
||
return FALSE;
|
||
}
|
||
|
||
Bool IsIndex(LogStruct *tmplg)
|
||
{
|
||
if (!StrCmp(tmplg->file,"/index.html") || !StrCmp(tmplg->file,"/"))
|
||
return TRUE;
|
||
else
|
||
return FALSE;
|
||
}
|
||
|
||
Bool IsKeeper(LogStruct *tmplg,CDate dstart,CDate dend)
|
||
{
|
||
if (dstart<=tmplg->datetime<=dend && !StrOcc(tmplg->file,'?') &&
|
||
StrLen(tmplg->file)>2 && 'A'<=tmplg->file[1]<='Z' &&
|
||
tmplg->size && tmplg->file[StrLen(tmplg->file)-1]!='/' &&
|
||
(StrLen(tmplg->file)<3 || MemCmp(&tmplg->file[1],"Wb",2)) &&
|
||
(StrLen(tmplg->file)<7 || MemCmp(&tmplg->file[1],"Family",6))) {
|
||
return TRUE;
|
||
|
||
} else
|
||
return FALSE;
|
||
}
|
||
|
||
Bool IPNumTreeAdd(LogStruct **_head,LogStruct *tmplg)
|
||
{
|
||
LogStruct *head;
|
||
if (UnusedStk<0x200) {
|
||
PrintErr("Stk Overflow");
|
||
throw;
|
||
}
|
||
if (head=*_head) {
|
||
if (tmplg->ip_num==head->ip_num)
|
||
return TRUE;
|
||
else if (tmplg->ip_num<head->ip_num)
|
||
return IPNumTreeAdd(&head->ip_num_left,tmplg);
|
||
else
|
||
return IPNumTreeAdd(&head->ip_num_right,tmplg);
|
||
} else {
|
||
tmplg->ip_num_left=NULL;
|
||
tmplg->ip_num_right=NULL;
|
||
*_head=tmplg;
|
||
return FALSE;
|
||
}
|
||
}
|
||
|
||
U0 LinkTreeAdd(LinkStruct **_root,LogStruct *tmplg)
|
||
{
|
||
I64 i;
|
||
LinkStruct *root,*tmplk;
|
||
if (UnusedStk<0x200) {
|
||
PrintErr("Stk Overflow");
|
||
throw;
|
||
}
|
||
if (root=*_root) {
|
||
if (!(i=StrCmp(tmplg->link,root->link)))
|
||
root->cnt++;
|
||
else if (i<0)
|
||
LinkTreeAdd(&root->left,tmplg);
|
||
else
|
||
LinkTreeAdd(&root->right,tmplg);
|
||
} else {
|
||
tmplk=CAlloc(sizeof(LinkStruct));
|
||
tmplk->link=tmplg->link;
|
||
tmplk->cnt=1;
|
||
*_root=tmplk;
|
||
}
|
||
}
|
||
|
||
U0 FileTreeAdd(LinkStruct **_root,LogStruct *tmplg)
|
||
{
|
||
I64 i;
|
||
LinkStruct *root,*tmplk;
|
||
if (UnusedStk<0x200) {
|
||
PrintErr("Stk Overflow");
|
||
throw;
|
||
}
|
||
if (root=*_root) {
|
||
if (!(i=StrCmp(tmplg->file,root->file)))
|
||
root->cnt++;
|
||
else if (i<0)
|
||
FileTreeAdd(&root->left,tmplg);
|
||
else
|
||
FileTreeAdd(&root->right,tmplg);
|
||
} else {
|
||
tmplk=CAlloc(sizeof(LinkStruct));
|
||
tmplk->file=tmplg->file;
|
||
tmplk->cnt=1;
|
||
*_root=tmplk;
|
||
}
|
||
}
|
||
|
||
U0 LinkTreeDel(LinkStruct *root)
|
||
{
|
||
if (root) {
|
||
LinkTreeDel(root->left);
|
||
LinkTreeDel(root->right);
|
||
Free(root);
|
||
}
|
||
}
|
||
|
||
U0 LinkTreeTraverse(LinkStruct *root)
|
||
{
|
||
if (root) {
|
||
LinkTreeTraverse(root->left);
|
||
"%3d:%$$Q\n",root->cnt,root->link;
|
||
LinkTreeTraverse(root->right);
|
||
}
|
||
}
|
||
|
||
U0 FileTreeDel(LinkStruct *root)
|
||
{
|
||
if (root) {
|
||
FileTreeDel(root->left);
|
||
FileTreeDel(root->right);
|
||
Free(root);
|
||
}
|
||
}
|
||
|
||
U0 FileTreeTraverse(LinkStruct *root)
|
||
{
|
||
if (root) {
|
||
FileTreeTraverse(root->left);
|
||
"%3d:%$$Q\n",root->cnt,root->file;
|
||
FileTreeTraverse(root->right);
|
||
}
|
||
}
|
||
|
||
U0 DownLoadRep(LogStruct *head,CDate dstart,CDate dend)
|
||
{
|
||
I64 i,j,cnt,dups,
|
||
hours_start,hours_end,*hour_cnts,*dup_cnts,
|
||
days_start,days_end,*day_cnts,*day_dup_cnts;
|
||
LogStruct *tmplg=head->next,*dup_head=NULL;
|
||
LinkStruct *link_root=NULL;
|
||
CDateStruct ds;
|
||
|
||
i=dstart*24;hours_start=i.u32[1];
|
||
i=dend *24;hours_end =i.u32[1];
|
||
|
||
days_start=(dstart+local_time_offset)>>32;
|
||
days_end =(dend+local_time_offset)>>32;
|
||
|
||
hour_cnts=CAlloc((hours_end-hours_start+1)*sizeof(I64));
|
||
dup_cnts =CAlloc((hours_end-hours_start+1)*sizeof(I64));
|
||
day_cnts =CAlloc((days_end-days_start+1)*sizeof(I64));
|
||
day_dup_cnts=CAlloc((days_end-days_start+1)*sizeof(I64));
|
||
dups=cnt=0;
|
||
while (tmplg!=head) {
|
||
if (IsKeeper(tmplg,dstart,dend) && IsDownLoad(tmplg)) {
|
||
i=tmplg->datetime*24;
|
||
hour_cnts[i.u32[1]-hours_start]++;
|
||
day_cnts[(tmplg->datetime+local_time_offset)>>32-days_start]++;
|
||
cnt++;
|
||
if (IPNumTreeAdd(&dup_head,tmplg)) {
|
||
day_dup_cnts[(tmplg->datetime+local_time_offset)>>32-days_start]++;
|
||
dup_cnts[i.u32[1]-hours_start]++;
|
||
dups++;
|
||
}
|
||
LinkTreeAdd(&link_root,tmplg);
|
||
}
|
||
tmplg=tmplg->next;
|
||
}
|
||
|
||
"\n\nDownloads of /TOS_Distro.ISO\n";
|
||
for (i=dstart;i<=dend;i+=1<<32)
|
||
"%D Dups:%5d Total:%5d Uniques:%5d\n",i,
|
||
day_dup_cnts[(i+local_time_offset)>>32-days_start],
|
||
day_cnts[(i+local_time_offset)>>32-days_start],
|
||
day_cnts[(i+local_time_offset)>>32-days_start]-
|
||
day_dup_cnts[(i+local_time_offset)>>32-days_start];
|
||
|
||
"\n\nDownloads of /TOS_Distro.ISO\n"
|
||
"'-' is a dup. '+' is not a dup.\n";
|
||
if (hours_end-hours_start>=HOURS_MAX)
|
||
i=hours_end-HOURS_MAX+1;
|
||
else
|
||
i=hours_start;
|
||
for (;i<=hours_end;i++) {
|
||
Date2Struct(&ds,i<<32/24+local_time_offset);
|
||
"%D %02d: ",i<<32/24,ds.hour;
|
||
for (j=0;j<dup_cnts[i-hours_start];j++)
|
||
'-';
|
||
for (;j<hour_cnts[i-hours_start];j++)
|
||
'+';
|
||
'\n';
|
||
}
|
||
"Total:%d Dups:%d Uniques:%d\n",cnt,dups,cnt-dups;
|
||
|
||
"\n\nDownloads of /TOS_Distro.ISO\n";
|
||
LinkTreeTraverse(link_root);
|
||
'\n';
|
||
|
||
LinkTreeDel(link_root);
|
||
Free(hour_cnts);
|
||
Free(dup_cnts);
|
||
Free(day_cnts);
|
||
Free(day_dup_cnts);
|
||
}
|
||
|
||
U0 FileRep(LogStruct *head,CDate dstart,CDate dend)
|
||
{
|
||
LogStruct *tmplg=head->next;
|
||
LinkStruct *file_root=NULL;
|
||
while (tmplg!=head) {
|
||
if (IsKeeper(tmplg,dstart,dend))
|
||
FileTreeAdd(&file_root,tmplg);
|
||
tmplg=tmplg->next;
|
||
}
|
||
"\n\nFile Hits\n";
|
||
FileTreeTraverse(file_root);
|
||
'\n';
|
||
FileTreeDel(file_root);
|
||
}
|
||
|
||
U0 IndexRep(LogStruct *head,CDate dstart,CDate dend)
|
||
{
|
||
I64 i,j,cnt,dups,
|
||
hours_start,hours_end,*hour_cnts,*dup_cnts,
|
||
days_start,days_end,*day_cnts,*day_dup_cnts;
|
||
LogStruct *tmplg=head->next,*dup_head=NULL;
|
||
LinkStruct *link_root=NULL;
|
||
CDateStruct ds;
|
||
|
||
i=dstart*24;hours_start=i.u32[1];
|
||
i=dend *24;hours_end =i.u32[1];
|
||
|
||
days_start=(dstart+local_time_offset)>>32;
|
||
days_end =(dend+local_time_offset)>>32;
|
||
|
||
hour_cnts=CAlloc((hours_end-hours_start+1)*sizeof(I64));
|
||
dup_cnts =CAlloc((hours_end-hours_start+1)*sizeof(I64));
|
||
day_cnts =CAlloc((days_end-days_start+1)*sizeof(I64));
|
||
day_dup_cnts=CAlloc((days_end-days_start+1)*sizeof(I64));
|
||
dups=cnt=0;
|
||
while (tmplg!=head) {
|
||
if (IsKeeper(tmplg,dstart,dend) && IsIndex(tmplg)) {
|
||
i=tmplg->datetime*24;
|
||
hour_cnts[i.u32[1]-hours_start]++;
|
||
day_cnts[(tmplg->datetime+local_time_offset)>>32-days_start]++;
|
||
cnt++;
|
||
if (IPNumTreeAdd(&dup_head,tmplg)) {
|
||
day_dup_cnts[(tmplg->datetime+local_time_offset)>>32-days_start]++;
|
||
dup_cnts[i.u32[1]-hours_start]++;
|
||
dups++;
|
||
}
|
||
LinkTreeAdd(&link_root,tmplg);
|
||
}
|
||
tmplg=tmplg->next;
|
||
}
|
||
|
||
"\n\nHits on /index.html\n"
|
||
"'-' is a dup. '+' is not a dup.\n";
|
||
for (i=dstart;i<=dend;i+=1<<32)
|
||
"%D Dups:%5d Total:%5d Uniques:%5d\n",i,
|
||
day_dup_cnts[(i+local_time_offset)>>32-days_start],
|
||
day_cnts[(i+local_time_offset)>>32-days_start],
|
||
day_cnts[(i+local_time_offset)>>32-days_start]-
|
||
day_dup_cnts[(i+local_time_offset)>>32-days_start];
|
||
|
||
"\n\nHits on /index.html\n";
|
||
if (hours_end-hours_start>=HOURS_MAX)
|
||
i=hours_end-HOURS_MAX+1;
|
||
else
|
||
i=hours_start;
|
||
for (;i<=hours_end;i++) {
|
||
Date2Struct(&ds,i<<32/24+local_time_offset);
|
||
"%D %02d: ",i<<32/24,ds.hour;
|
||
for (j=0;j<dup_cnts[i-hours_start];j++)
|
||
'-';
|
||
for (;j<hour_cnts[i-hours_start];j++)
|
||
'+';
|
||
'\n';
|
||
}
|
||
"Total:%d Dups:%d Uniques:%d\n",cnt,dups,cnt-dups;
|
||
|
||
"\n\nHits on /index.html\n";
|
||
LinkTreeTraverse(link_root);
|
||
'\n';
|
||
|
||
LinkTreeDel(link_root);
|
||
Free(hour_cnts);
|
||
Free(dup_cnts);
|
||
Free(day_cnts);
|
||
Free(day_dup_cnts);
|
||
}
|
||
|
||
U0 WebLogRep(U8 *mask,U8 *output_filename)
|
||
{
|
||
LogStruct *head;
|
||
CDate dstart=I64_MAX,dend=I64_MIN;
|
||
|
||
DocMax;
|
||
|
||
head=PrsLogFiles(mask,&dstart,&dend);
|
||
if (dstart>dend)
|
||
PrintErr("No Data.\n");
|
||
else {
|
||
dstart=GetDate("Start(%D):",dstart);
|
||
dend =GetDate("End (%D):",dend);
|
||
BlockIPNuip(head);
|
||
|
||
DocClear;
|
||
"$$WW,0$$";
|
||
IndexRep(head,dstart,dend);
|
||
FileRep(head,dstart,dend);
|
||
DownLoadRep(head,dstart,dend);
|
||
|
||
StrCpy(DocPut->filename.name,output_filename);
|
||
DocWrite(DocPut,TRUE);
|
||
|
||
"$$WW,1$$";
|
||
}
|
||
LogLstDel(head);
|
||
}
|
||
|
||
#if __CMD_LINE__
|
||
Cd(__DIR__);;
|
||
WebLogRep("*.log*","~/DemoWebLog.DD.Z");
|
||
#endif
|