文章目录

• 1. 数据挖掘：/etc/rc.local，sudo su
• 2.HTTP协议：优先wget
• 3.非结构化数据存储：blob，pzhrain24file
• 4.数据管理子系统：数据字典表
• 5.监控告警子系统：tmpfs，top，/proc/stat，/proc/meminfo，mount ，processes
• • 5.1 du，df，fdisk，lsblk：磁盘=硬盘
  • 5.2 文件：建表入表，传输

---

1. 数据挖掘：/etc/rc.local，sudo su

数据挖掘dmining和数据交换exptables差不多（exptables自己设计数据表有keyid字段，从其他地方拿数据未必有keyid字段需要指定，也就是数据是增量的，但不提供增量提取方式即没有keyid字段），将数据从数据库中取出导入文件（执行一个sql拿出数据）。vi /etc/rc.local（随操作系统自启动）。

vi /htidc/gz…sh上面如下，如下要写的其实就是< selectsql >字段里内容。比如取近一小时数据，每次取到都有重复，但在入库时（获取到的数据生成xml再入自己库）处理重复就行。

//dminoracle.cpp #include "_public.h" #include "_ooci.h" //oracle数据库是_ooci.h,mysql数据库换成_mysql.h,PostgreSQL数据库换成_postgresql.h //公安内外网协议不通，数据挖掘不一定有数据，所以写入文件。 struct st_arg // 主程序的参数 {char connstr[101];char charset[51];char tname[51];char cols[1001];char fieldname[1001];char fieldlen[501];int exptype;char andstr[501];char bname[51];char ename[51];char idfieldname[51];char idfilename[301];char exppath[301];int timetvl; } starg; CLogFile logfile; connection conn; bool _dmintables(); void EXIT(int sig); vector<string> vfieldname; // 存放拆分fieldname后的容器 vector<int> vfieldlen; // 存放拆分fieldlen后的容器 int maxfieldlen; // 存放fieldlen的最大值 void SplitFields(); // 拆分fieldname和fieldlen void _help(char *argv[]); long maxkeyid; // 已挖掘数据的最大的id bool LoadMaxKeyid(); // 从记录已获取数据最大id的文件中加载已挖掘数据的最大的id bool UptMaxKeyid(); // 更新已挖掘数据的最大的id到文件中 bool _xmltoarg(char *strxmlbuffer);int main(int argc,char *argv[]) {if (argc!=3) { _help(argv); return -1; }CloseIOAndSignal();signal(SIGINT,EXIT); signal(SIGTERM,EXIT);if (logfile.Open(argv[1],"a+")==false){printf("打开日志文件失败（%s）。\n",argv[1]); return -1;} if (_xmltoarg(argv[2])==false) return -1; //把xml解析到参数starg结构中while (true){if (conn.connecttodb(starg.connstr,starg.charset) != 0){logfile.Write("connect database %s failed.\n",starg.connstr); sleep(starg.timetvl); continue;}// logfile.Write("export table %s.\n",starg.tname);if (_dmintables() == false) logfile.Write("export tables failed.\n");conn.disconnect(); // 断开与数据库的连接sleep(starg.timetvl);}return 0; } void EXIT(int sig) {logfile.Write("程序退出，sig=%d\n\n",sig);exit(0); }//11111111111111111111111111111111111111111111显示程序的帮助 void _help(char *argv[]) {printf("\n");printf("Using:/htidc/public/bin/dminoracle logfilename xmlbuffer\n\n");printf("增量挖掘示例：\n");printf("Sample:/htidc/public/bin/dminoracle /log/shqx/dminoracle_surfdata_from_qx.log \"<connstr>shqx/pwdidc@snorcl11g_198</connstr><charset>Simplified Chinese_China.ZHS16GBK</charset><tname>T_SURFDATA</tname><cols>obtid,to_char(ddatetime,'yyyymmddhh24miss'),t,p,u,wd,wf,r,vis</cols><fieldname>obtid,ddatetime,t,p,u,wd,wf,r,vis</fieldname><fieldlen>5,14,8,8,8,8,8,8,8</fieldlen><exptype>1</exptype><andstr> and obtid in ('59293','50745')</andstr><bname>SURFDATA_</bname><ename>_from_qx</ename><idfilename>/data/dmin/SURFDATA_from_qx.txt</idfilename><idfieldname>keyid</idfieldname><exppath>/data/shqx/sdata/fromqx</exppath><timetvl>30</timetvl>\"\n\n");printf("全量挖掘示例：\n");printf("Sample:/htidc/public/bin/dminoracle /log/shqx/dminoracle_obtcode_from_qx.log \"<connstr>shqx/pwdidc@snorcl11g_198</connstr><charset>Simplified Chinese_China.ZHS16GBK</charset><tname>T_OBTCODE</tname><cols>obtid,obtname,provname,lat,lon,height</cols><fieldname>obtid,obtname,provname,lat,lon,height</fieldname><fieldlen>5,30,30,8,8,8</fieldlen><exptype>2</exptype><andstr> and rsts=1 and obtid in ('59293','50745')</andstr><bname>OBTCODE_</bname><ename>_from_qx</ename><exppath>/data/shqx/sdata/fromqx</exppath><timetvl>300</timetvl>\"\n\n");printf("本程序是数据中心的公共功能模块，从其它业务系统的数据库中挖掘数据，用于入库到数据中心。\n");printf("logfilename是本程序运行的日志文件。\n");printf("xmlbuffer为文件传输的参数，如下：\n");printf("数据库的连接参数 <connstr>shqx/pwdidc@snorcl11g_198</connstr>\n");printf("数据库的字符集 <charset>Simplified Chinese_China.ZHS16GBK</charset> 这个参数要与数据源数据库保持>一致，否则会出现中文乱码的情况。\n");printf("待挖掘数据的表名 <tname>T_SURFDATA</tname>\n");printf("需要挖掘字段的列表 <cols>obtid,to_char(ddatetime,'yyyymmddhh24miss'),t,p,u,wd,wf,r,vis</cols> 可以采用函数。\n");printf("挖掘字段的别名列表 <fieldname>obtid,ddatetime,t,p,u,wd,wf,r,vis</fieldname> 必须与cols一一对应。\n");printf("挖掘字段的长度列表 <fieldlen>5,14,8,8,8,8,8,8,8</fieldlen> 必须与cols一一对应。\n");printf("挖掘数据的方式 <exptype>1</exptype> 1-增量挖掘；2-全量挖掘，如果是增量挖掘，要求表一定要有表达记录序号的id字段。\n");printf("挖掘数据的附加条件 <andstr> and obtid in ('59293','50745')</andstr> 注意，关键字and不能少。\n");printf("数据文件的命名的前部分 <bname>SURFDATA_</bname>\n");printf("数据文件的命名的后部分 <ename>_from_qx</ename>\n");printf("挖掘数据表记录号字段名 <idfieldname>keyid</idfieldname> 当exptype=1时该参数有效。\n");printf("已挖掘数据id保存的文件名 <idfilename>/data/dmin/SURFDATA_from_qx.txt</idfilename> 当exptype=1时该参数有效。\n");printf("挖掘文件存放的目录 <exppath>/data/shqx/sdata/fromqx</exppath>\n");printf("挖掘数据的时间间隔 <timetvl>30</timetvl> 单位：秒，建议大于10。\n");printf("以上参数，除了idfieldname、idfilename和andstr，其它字段都不允许为空。\n\n\n"); }//1111111111111111111111111111111111111111111111把xml解析到参数starg结构中 bool _xmltoarg(char *strxmlbuffer) {memset(&starg,0,sizeof(struct st_arg));GetXMLBuffer(strxmlbuffer,"connstr",starg.connstr);if (strlen(starg.connstr)==0) { logfile.Write("connstr is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"charset",starg.charset);if (strlen(starg.charset)==0) { logfile.Write("charset is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"tname",starg.tname);if (strlen(starg.tname)==0) { logfile.Write("tname is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"cols",starg.cols);if (strlen(starg.cols)==0) { logfile.Write("cols is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"fieldname",starg.fieldname);if (strlen(starg.fieldname)==0) { logfile.Write("fieldname is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"fieldlen",starg.fieldlen);if (strlen(starg.fieldlen)==0) { logfile.Write("fieldlen is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"exptype",&starg.exptype);if ( (starg.exptype!=1) && (starg.exptype!=2) ) { logfile.Write("exptype is not in (1,2).\n"); return false; }GetXMLBuffer(strxmlbuffer,"andstr",starg.andstr);if (strlen(starg.andstr)==0) { logfile.Write("andstr is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"bname",starg.bname);if (strlen(starg.bname)==0) { logfile.Write("bname is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"ename",starg.ename);if (strlen(starg.ename)==0) { logfile.Write("ename is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"idfieldname",starg.idfieldname);if ( (starg.exptype==1) && (strlen(starg.idfieldname)==0) ) { logfile.Write("idfieldname is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"idfilename",starg.idfilename);if ( (starg.exptype==1) && (strlen(starg.idfilename)==0) ) { logfile.Write("idfilename is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"exppath",starg.exppath);if (strlen(starg.exppath)==0) { logfile.Write("exppath is null.\n"); return false; }GetXMLBuffer(strxmlbuffer,"timetvl",&starg.timetvl);if (starg.timetvl==0) { logfile.Write("timetvl is null.\n"); return false; }// 拆分fieldname和fieldlenSplitFields();// 判断fieldname和fieldlen中元素的个数一定要相同if (vfieldname.size() != vfieldlen.size() ) { logfile.Write("fieldname和fieldlen的元素个数不同。.\n"); return false; }return true; }//11111111111111111111111111111111111本程序的业务流程主函数 bool _dmintables() {// 从记录已获取数据最大id的文件中加载已挖掘数据的最大的idif (LoadMaxKeyid()==false) { logfile.Write("LoadMaxKeyid() failed.\n"); return false; }// 生成挖掘数据的SQL语句char strsql[4096]; char fieldvalue[vfieldname.size()][maxfieldlen+1]; // 输出变量定义为一个二维数组memset(strsql,0,sizeof(strsql));if (starg.exptype==1)sprintf(strsql,"select %s,%s from %s where 1=1 and %s>%ld %s order by %s",starg.cols,starg.idfieldname,starg.tname,starg.idfieldname,maxkeyid,starg.andstr,starg.idfieldname);elsesprintf(strsql,"select %s from %s where 1=1 %s",starg.cols,starg.tname,starg.andstr);sqlstatement stmt(&conn);stmt.prepare(strsql);for (int ii=0;ii<vfieldname.size();ii++){stmt.bindout(ii+1,fieldvalue[ii],vfieldlen[ii]);} if (starg.exptype==1) stmt.bindout(vfieldname.size()+1,&maxkeyid); //如果是增量挖掘，还要绑定id字段if (stmt.execute() != 0) // 执行挖掘数据的SQL{logfile.Write("select %s failed.\n%s\n%s\n",starg.tname,stmt.m_cda.message,stmt.m_sql); return false;}int iFileSeq=1; // 待生成文件的序号char strFileName[301],strLocalTime[21];CFile File;while (true){memset(fieldvalue,0,sizeof(fieldvalue)); if (stmt.next() !=0) break; if (File.IsOpened()==false) // 把数据写入文件{memset(strLocalTime,0,sizeof(strLocalTime));LocalTime(strLocalTime,"yyyymmddhh24miss");memset(strFileName,0,sizeof(strFileName));sprintf(strFileName,"%s/%s%s%s_%d.xml",starg.exppath,starg.bname,strLocalTime,starg.ename,iFileSeq++);if (File.OpenForRename(strFileName,"w")==false){logfile.Write("File.OpenForRename(%s) failed.\n",strFileName); return false;}File.Fprintf("<data>\n");}for (int ii=0;ii<vfieldname.size();ii++){File.Fprintf("<%s>%s</%s>",vfieldname[ii].c_str(),fieldvalue[ii],vfieldname[ii].c_str());}File.Fprintf("<endl/>\n");//1000条记录写入一个文件完成if (stmt.m_cda.rpc%1000==0) {File.Fprintf("</data>\n");if (File.CloseAndRename()==false){logfile.Write("File.CloseAndRename(%s) failed.\n",strFileName); return false;}// 更新已挖掘数据的最大的id到文件中if (UptMaxKeyid()==false) { logfile.Write("UptMaxKeyid() failed.\n"); return false; }logfile.Write("create file %s ok.\n",strFileName);}}//不够1000条的写入一个文件if (File.IsOpened()==true){File.Fprintf("</data>\n");if (File.CloseAndRename()==false){logfile.Write("File.CloseAndRename(%s) failed.\n",strFileName); return false;}// 更新已挖掘数据的最大的id到文件中if (UptMaxKeyid()==false) { logfile.Write("UptMaxKeyid() failed.\n"); return false; }logfile.Write("create file %s ok.\n",strFileName);}if (stmt.m_cda.rpc>0) logfile.Write("本次挖掘了%d条记录。\n",stmt.m_cda.rpc);return true; }//111111111111111111111111111111111111拆分fieldname和fieldlen void SplitFields() {vfieldname.clear(); vfieldlen.clear(); maxfieldlen=0; CCmdStr CmdStr;CmdStr.SplitToCmd(starg.fieldname,",");vfieldname.swap(CmdStr.m_vCmdStr);int ifieldlen=0;CmdStr.SplitToCmd(starg.fieldlen,",");for (int ii=0;ii<CmdStr.CmdCount();ii++){ CmdStr.GetValue(ii,&ifieldlen);if (ifieldlen>maxfieldlen) maxfieldlen=ifieldlen; // 得到fieldlen的最大值vfieldlen.push_back(ifieldlen);} }//111111111111111111111111111111从记录已获取数据最大id的文件中加载已挖掘数据的最大的id bool LoadMaxKeyid() {if (starg.exptype!=1) return true;CFile File;if (File.Open(starg.idfilename,"r")==false) {logfile.Write("注意，%s文件不存在，程序将从新开始挖掘数据。\n",starg.idfilename); return true;}char strBuf[21];memset(strBuf,0,sizeof(strBuf));File.Fread(strBuf,20);maxkeyid=atol(strBuf);logfile.Write("maxkeyid=%d\n",maxkeyid);return true; }//11111111111111111111111111111111更新已挖掘数据的最大的id到文件中 bool UptMaxKeyid() {if (starg.exptype!=1) return true;CFile File;if (File.Open(starg.idfilename,"w")==false) {logfile.Write("File.Open(%s) failed.\n",starg.idfilename); return false;}File.Fprintf("%ld",maxkeyid);return true; }

2.HTTP协议：优先wget

数据共享平台也有一个web系统（服务端）用java做的，vi /etc/rc.local如下。

如下http客户端访问不了，startup后可访问。

按照http格式向网站发报文也会得到回应，如下http客户端。


wgetclient是用wget命令下载，httpclient下载的内容有时候会有断行，wget下载不会。优先wget，wget下载不了的用http。wget支持http，https，ftp，sftp等协议。linux下同步文件rsync，curl，wget，ftp，sftp。

如上调用接口，如下还可以右击网页上图片复制图片地址。

程序在后台跑，有新图就拿下来，wgetclient将网页内容全弄下来，搞清图片命名规律进行解析。

//wgetclient.cpp #include "_public.h" void EXIT(int sig); CLogFile logfile; int main(int argc, char *argv[]) {if(argc!=6){printf("Usage:%s weburl tmpfilename outputfilename logfilename charset\n",argv[0]); printf("本程序用于获取WEB网页的内容。\n");printf("weburl 网页WEB的地址。\n");printf("tmpfilename 获取到的网页的内容存放的全路径的临时文件名，该文件可能是utf-8或其它编码。\n");printf("outputfilename 最终的输出文件全路径文件名，该文件是gb18030编码，注意tmpfilename被转换为outputfilename后，tmpfilename文件被自动删除。\n");printf("logfilename 本程序的运行产生的日志文件名。\n");printf("charset 网页的字符集，如utf-8\n\n");exit(1);}// 关闭全部的信号和输入输出// 设置信号,在shell状态下可用 "kill + 进程号" 正常终止些进程// 但请不要用 "kill -9 +进程号" 强行终止CloseIOAndSignal(); signal(SIGINT,EXIT); signal(SIGTERM,EXIT);if (logfile.Open(argv[4],"a+") == false){printf("logfile.Open(%s) failed.\n",argv[4]); return -1;}MKDIR(argv[2],true); MKDIR(argv[3],true);char strweburl[3001];memset(strweburl,0,sizeof(strweburl));strncpy(strweburl,argv[1],3000);char strcmd[3024];memset(strcmd,0,sizeof(strcmd));snprintf(strcmd,3000,"/usr/bin/wget -c -q -O %s \"%s\" 1>>/dev/null 2>>/dev/null",argv[2],strweburl);system(strcmd);logfile.Write("%s\n",strcmd);char strfilenametmp[301];memset(strfilenametmp,0,sizeof(strfilenametmp));snprintf(strfilenametmp,300,"%s.tmp",argv[3]);// 把获取到的网页转换为中文memset(strcmd,0,sizeof(strcmd));snprintf(strcmd,256,"iconv -c -f %s -t gb18030 %s -o %s",argv[5],argv[2],strfilenametmp);system(strcmd);logfile.Write("%s\n",strcmd);REMOVE(argv[2]); // 删除临时文件 RENAME(strfilenametmp,argv[3]);return 0; }void EXIT(int sig) {if (sig > 0) signal(sig,SIG_IGN);logfile.Write("catching the signal(%d).\n",sig);logfile.Write("wgetclient exit.\n");exit(0); }

//wgetrain24.cpp #include "_public.h" void EXIT(int sig); CLogFile logfile; bool GetURL(char *strBuffer,char *strURL,char *strFileName); int main(int argc, char *argv[]) {if(argc!=4){printf("Usage:%s logfilename tmpfilename outputfilename\n",argv[0]); printf("Sample:./wgetrain24 /log/shqx/wgetrain24.log /data/wgettmp /data/wfile/zhrain24\n\n");printf("本程序用于从中国天气网获取逐小时降雨量实况图。\n");printf("中国天气网的url是http://products.weather.com.cn/product/Index/index/procode/JC_JSL_ZH.shtml\n");printf("如果中国天气网的url改变，程序也在做改动。\n");printf("logfilename 本程序的运行产生的日志文件名。\n");printf("tmpfilename 本程序运行产生的临时文件存放的目录。\n");printf("获取逐小时降雨量实况图存放的目录。\n\n");exit(1);}CloseIOAndSignal(); signal(SIGINT,EXIT); signal(SIGTERM,EXIT);if (logfile.Open(argv[1],"a+") == false){printf("logfile.Open(%s) failed.\n",argv[1]); return -1;}MKDIR(argv[2],false); MKDIR(argv[3],false);while (true){ char strwgetclient[2001]; // 调用wgetclient获取网页内容memset(strwgetclient,0,sizeof(strwgetclient));snprintf(strwgetclient,2000,"/htidc/public/bin/wgetclient \"http://products.weather.com.cn/product/Index/index/procode/JC_JSL_ZH.shtml\" %s/wgetclient_%d.tmp %s/wgetclient_%d.html %s/wgetclient.log utf-8",argv[2],getpid(),argv[2],getpid(),argv[2]);system(strwgetclient);// logfile.Write("%s\n",strwgetclient);// 打开网页内容文件char stroutputfile[301];memset(stroutputfile,0,sizeof(stroutputfile));snprintf(stroutputfile,300,"%s/wgetclient_%d.html",argv[2],getpid());CFile File;if (File.Open(stroutputfile,"r")==false){logfile.Write("File.Open(%s) failed.\n",stroutputfile); sleep(60); continue;} char strBuffer[1001],strURL[501],strFullFileName[301],strFileName[101]; // 得到全部的图片文件名while (true){memset(strBuffer,0,sizeof(strBuffer));memset(strURL,0,sizeof(strURL));memset(strFullFileName,0,sizeof(strFullFileName));memset(strFileName,0,sizeof(strFileName)); if (File.Fgets(strBuffer,1000)==false) break; if (MatchFileName(strBuffer,"*PWCP_TWC_WEAP_SFER_ER1_TWC_L88_P9_20*.JPG*")==false) continue; // logfile.Write("%s",strBuffer); if (GetURL(strBuffer,strURL,strFileName)==false) continue; //解析出url和文件名 snprintf(strFullFileName,300,"%s/%s",argv[3],strFileName); //文件已存在，不采集if (access(strFullFileName,F_OK)==0) continue; logfile.Write("download %s ",strFileName); //调用wget获取文件memset(strwgetclient,0,sizeof(strwgetclient));snprintf(strwgetclient,500,"wget \"%s\" -o %s/wgetrain24.log -O %s",strURL,argv[2],strFullFileName);system(strwgetclient); if (access(strFullFileName,F_OK)==0) logfile.WriteEx("ok.\n");else logfile.WriteEx("failed.\n");} File.CloseAndRemove();sleep(60);}return 0; }bool GetURL(char *strBuffer,char *strURL,char *strFileName) {char *start,*end;start=end=0;if ((start=strstr(strBuffer,"http"))==0) return false;if ((end=strstr(start,"\""))==0) return false; //找双引号strncpy(strURL,start,end-start);strcpy(strFileName,strstr(strURL,"PWCP"));return true; } void EXIT(int sig) {if (sig > 0) signal(sig,SIG_IGN);logfile.Write("catching the signal(%d).\n",sig);logfile.Write("wgetclient exit.\n");exit(0); }

3.非结构化数据存储：blob，pzhrain24file

// 本程序演示如何把磁盘文件的文本文件写入Oracle的BLOB字段中。 //filetoblob.cpp，实时生成的不要存oracle的blob字段 #include "_ooci.h" int main(int argc,char *argv[]) {connection conn; // 连接数据库，返回值0-成功，其它-失败// 失败代码在conn.m_cda.rc中，失败描述在conn.m_cda.message中。if (conn.connecttodb("scott/tiger@snorcl11g_198","Simplified Chinese_China.ZHS16GBK") != 0){printf("connect database %s failed.\n%s\n","scott/tiger@orcl",conn.m_cda.message); return -1;} sqlstatement stmt(&conn); // SQL语言操作类 // 为了方便演示，把goods表中的记录全删掉，再插入一条用于测试的数据。// 不需要判断返回值stmt.prepare("\BEGIN\delete from goods;\insert into goods(id,name,pic) values(1,'商品名称',empty_blob());\END;"); // 执行SQL语句，一定要判断返回值，0-成功，其它-失败。if (stmt.execute() != 0){printf("stmt.execute() failed.\n%s\n%s\n",stmt.m_sql,stmt.m_cda.message); return -1;}// 使用游标从goods表中提取id为1的pic字段// 注意了，同一个sqlstatement可以多次使用// 但是，如果它的sql改变了，就要重新prepare和bindin或bindout变量stmt.prepare("select pic from goods where id=1 for update");stmt.bindblob();// 执行SQL语句，一定要判断返回值，0-成功，其它-失败。if (stmt.execute() != 0){printf("stmt.execute() failed.\n%s\n%s\n",stmt.m_sql,stmt.m_cda.message); return -1;}// 获取一条记录，一定要判断返回值，0-成功，1403-无记录，其它-失败。if (stmt.next() != 0) return 0; // 把磁盘文件pic_in.jpg的内容写入BLOB字段，一定要判断返回值，0-成功，其它-失败。if (stmt.filetolob((char *)"pic_in.jpg") != 0){printf("stmt.filetolob() failed.\n%s\n",stmt.m_cda.message); return -1;} conn.commit(); // 提交事务return 0; }

// pzhrain24file.cpp #include "_public.h" #include "_ooci.h" #include "_shqx.h" CLogFile logfile; CDir Dir; // 处理数据文件 bool _pzhrain24file(char *strargv2,char *strargv4,char *strargv5); connection conn; void EXIT(int sig);int main(int argc,char *argv[]) {if (argc!=7){printf("\n本程序用于处理全国逐小时雨量实况图片文件。\n\n");printf("/htidc/shqx/bin/pzhrain24file logfilename connstr srcpathname dstpathname tname timetvl\n");printf("例如：/htidc/shqx/bin/pzhrain24file /log/shqx/pzhrain24file.log shqx/pwdidc@snorcl11g_198 /data/wfile/zhrain24 /qxfile/zhrain24 T_ZHRAIN24 30\n");printf("logfilename 本程序运行的日志文件名。\n");printf("connstr 数据库的连接参数。\n");printf("srcpathname 原始文件存放的目录，文件命名如PWCP_TWC_WEAP_SFER_ER1_TWC_L88_P9_20191101070000000.JPG。\n");printf("dstpathname 目标文件存放的目录，文件按yyyy/mm/dd组织目录，重命名为zhrain24_yyyymmddhh24miss.jpg。\n");printf("tname 数据存放的表名。\n");printf("timetvl 本程序运行的时间间隔，单位：秒。\n");return -1;}CloseIOAndSignal();signal(SIGINT,EXIT); signal(SIGTERM,EXIT);if (logfile.Open(argv[1],"a+")==false){printf("打开日志文件失败（%s）。\n",argv[1]); return -1;}logfile.Write("程序启动。\n");while (true){// logfile.Write("开始扫描目录。\n");// 扫描数据文件存放的目录，只匹配"PWCP_TWC_WEAP_SFER_ER1_TWC_L88_P9_20*.JPG"if (Dir.OpenDir(argv[3],"PWCP_TWC_WEAP_SFER_ER1_TWC_L88_P9_20*.JPG",1000,true,true)==false){logfile.Write("Dir.OpenDir(%s) failed.\n",argv[3]); sleep(atoi(argv[6])); continue;} while (true) // 逐个处理目录中的数据文件{if (Dir.ReadDir()==false) break;if (_pzhrain24file(argv[2],argv[4],argv[5])==false) {logfile.WriteEx("失败。\n"); continue;}}if (conn.m_state==1) conn.disconnect(); sleep(atoi(argv[6]));}return 0; } void EXIT(int sig) {logfile.Write("程序退出，sig=%d\n\n",sig);exit(0); }//1111111111111111111111111111111111111111111处理数据文件 bool _pzhrain24file(char *strargv2,char *strargv4,char *strargv5) {char strddatetime[21]; // 文件的数据时间，格式yyyymmddhh24missmemset(strddatetime,0,sizeof(strddatetime));strncpy(strddatetime,strstr(Dir.m_FileName,"20"),14);//搜索文件名PWCP_TWC…中20，后面取14位，重命名为zhrain24_%s.jpgchar strdstfilename[301]; // 目标文件名，不带路径memset(strdstfilename,0,sizeof(strdstfilename));snprintf(strdstfilename,300,"zhrain24_%s.jpg",strddatetime);char strdstfilepath[301]; // 目标文件存放的目录memset(strdstfilepath,0,sizeof(strdstfilepath));snprintf(strdstfilepath,300,"%s/",strargv4);strncat(strdstfilepath,strddatetime,4); strcat(strdstfilepath,"/"); // 年的子目录strncat(strdstfilepath,strddatetime+4,2); strcat(strdstfilepath,"/"); // 月的子目录strncat(strdstfilepath,strddatetime+6,2); strcat(strdstfilepath,"/"); // 日的子目录char strfulldstfilename[301]; // 目标文件名，全路径memset(strfulldstfilename,0,sizeof(strfulldstfilename));snprintf(strfulldstfilename,300,"%s%s",strdstfilepath,strdstfilename);// 如果文件已处理(目标文件已存在)，直接返回成功。if (access(strfulldstfilename,F_OK) == 0) return true;if (conn.m_state==0){if (conn.connecttodb(strargv2,"Simplified Chinese_China.ZHS16GBK")!=0){logfile.Write("connect database(%s) failed.\n%s\n",strargv2,conn.m_cda.message); return false;}// logfile.Write("连接数据库成功。\n");}// 把源文件复制到目标文件if (COPY(Dir.m_FullFileName,strfulldstfilename)==false) {logfile.Write("复制文件COPY(%s,%s)...failed.\n",Dir.m_FullFileName,strfulldstfilename); return false;}// 把非结构化数据文件写入oracle数据库的表中if (FileToTable(&conn,&logfile,strargv5,strfulldstfilename,strddatetime)!=0){logfile.Write("把文件%s存入%s...failed.\n",strfulldstfilename,strargv5); return false;}logfile.Write("把文件%s存入%s...ok.\n",strfulldstfilename,strargv5); return true; }

4.数据管理子系统：数据字典表

Oracle数据字典：是一组表和视图的结构（就像仓库里有什么，每种有多少，哪些空间可用等日记）。自己创建的表，oracle会将这表的信息自动写入系统表，系统表上再创建一视图供客户查看。数据字典中的表是不能被访问修改（自己用的），但是可以访问数据字典的视图就可以知道自己创建的表的详细信息。


tab也是视图，tab里面表，视图，同义词都可以查到。若是select * from dba_tables；必须是dba用户才可以查，三类user_，all_，dba_

数据量大了会出现性能和数据管理，迁移，备份问题。假如有1亿数据量，不能用exp导出，一个文件有几百G导出要一天。索引设计要合理，不然table scan跑不动。一亿以下的数据单表存放，一亿以上的数据考虑分表，十亿虽然比一亿容量大十倍，但性能不会下降十倍，因为索引（比如按首字母找名字）。

如下并行数据库分布式查询可解决多表性能低的问题。

写一些程序实现数据管理，数据删除（deletables.cpp）或放历史表等等。三种表数据结构一样，迁移数据可写成一个通用功能模块程序。若是不同表，只知道表名去查询数据字典得到数据结构。如下hsmtable.cpp（和deletetable.cpp像）：数据源表就是要把数据从哪个表迁移出来，列名从数据字典取出，把列拼成一个字符串，获取列名后把数据rowid从源表查出来，生成插入目的表sql，删除源表sql。

//hsmtable.cpp #include "_public.h" #include "_ooci.h" char logfilename[301]; char connstr[101]; char srctname[51]; char dsttname[51]; char where[1024]; char hourstr[101]; char localhour[21]; int maxcount=1; connection conn; CLogFile logfile; void EXIT(int sig); char strColumnStr[2048]; //获取dsttname表全部的列 bool GetColumnStr(); void _help(char *argv[]); bool _hsmtables();int main(int argc,char *argv[]) {if (argc != 2) { _help(argv); return -1; }memset(logfilename,0,sizeof(logfilename));memset(connstr,0,sizeof(connstr));memset(srctname,0,sizeof(srctname));memset(dsttname,0,sizeof(dsttname));memset(where,0,sizeof(where));memset(hourstr,0,sizeof(hourstr));GetXMLBuffer(argv[1],"logfilename",logfilename,300);GetXMLBuffer(argv[1],"connstr",connstr,100);GetXMLBuffer(argv[1],"srctname",srctname,50);GetXMLBuffer(argv[1],"dsttname",dsttname,50);GetXMLBuffer(argv[1],"where",where,1000);GetXMLBuffer(argv[1],"maxcount",&maxcount);GetXMLBuffer(argv[1],"hourstr",hourstr,2000);if (strlen(logfilename) == 0) { printf("logfilename is null.\n"); return -1; }if (strlen(connstr) == 0) { printf("connstr is null.\n"); return -1; }if (strlen(srctname) == 0) { printf("srctname is null.\n"); return -1; }if (strlen(dsttname) == 0) { printf("dsttname is null.\n"); return -1; }if (strlen(where) == 0) { printf("where is null.\n"); return -1; }if ( (maxcount<1) || (maxcount>1000) ) { printf("maxcount %d is invalid,should in 1-1000.\n",maxcount); return -1; }if (strlen(hourstr) == 0) { printf("hourstr is null.\n"); return -1; }CloseIOAndSignal();signal(SIGINT,EXIT); signal(SIGTERM,EXIT);if (logfile.Open(logfilename,"a+") == false){printf("logfile.Open(%s) failed.\n",logfilename); return -1;}while (true){ memset(localhour,0,sizeof(localhour)); //判断当前时间是否在启动时间之内LocalTime(localhour,"hh24");if (strstr(hourstr,localhour)==0) { sleep(60); continue; }if (conn.connecttodb(connstr,"Simplified Chinese_China.ZHS16GBK") != 0){logfile.Write("connect database %s failed.\n",connstr); sleep(60); continue; }logfile.Write("from table %s to %s.\n",srctname,dsttname);if (_hsmtables() == false) logfile.Write("_hsmtables failed.\n"); conn.disconnect();sleep(60); }return 0; } void EXIT(int sig) {printf("程序退出，sig=%d\n\n",sig);exit(0); }//111111111111111111111111111111111111111111111111111111111111111 void _help(char *argv[]) {printf("\nUsing:/htidc/public/bin/hsmtables \"<logfilename>/log/shqx/hsmtables_SURFDATA.log</logfilename><connstr>shqx/pwdidc@snorcl11g_198</connstr><srctname>T_SURFDATA</srctname><dsttname>T_SURFDATA_HIS</dsttname><where>where ddatetime<sysdate-10</where><maxcount>500</maxcount><hourstr>23,01,02,03,04,05,06</hourstr>\"\n\n");printf("这是一个工具程序，用于清理表中的数据。\n");printf("<logfilename>/log/shqx/hsmtables_SURFDATA.log</logfilename> 本程序运行日志文件名。\n");printf("<connstr>szidc/pwdidc@SZQX_10.153.97.251</connstr> 目的数据库的连接参数。\n");printf("<srctname>T_SURFDATA</srctname> 数据源表名。\n");printf("<dsttname>T_SURFDATA_HIS</dsttname> 目的数据表名。\n");printf("<where>where ddatetime<sysdate-10</where> 待迁移数据的条件。\n");printf("<maxcount>500</maxcount> 单次执行数据迁移的记录数，取值在1-1000之间。\n");printf("<hourstr>23,01,02,03,04,05,06</hourstr> 本程序启动的时次，小时，时次之间用半角的逗号分隔开。\n\n");return; }//11111111111111111111111111111111111111111111111111111111111111 bool _hsmtables() { if (GetColumnStr() == false) return false; //获取dsttname表全部的列int ccount=0;char strrowid[51],strrowidn[maxcount][51];sqlstatement selstmt(&conn); //把数据的rowid从srctname表中查出来selstmt.prepare("select rowid from %s %s",srctname,where);selstmt.bindout(1, strrowid,50);if (selstmt.execute() != 0){logfile.Write("%s failed.\n%s\n",selstmt.m_sql,selstmt.m_cda.message); return false;}int ii=0; // 生成插入dsttname表和删除srctname表的SQLchar strInsertSQL[10241],strDeleteSQL[10241];memset(strInsertSQL,0,sizeof(strInsertSQL));memset(strDeleteSQL,0,sizeof(strDeleteSQL));sprintf(strInsertSQL,"insert into %s(%s) select %s from %s where rowid in (",dsttname,strColumnStr,strColumnStr,srctname);//%s(%s)里面的%s是字段名，字段顺序没必要保持一致，所有把列名从数据字典取出，目的表字段比原表少，只备份一部分字段sprintf(strDeleteSQL,"delete from %s where rowid in (",srctname);char strtemp[11];for (ii=0; ii<maxcount; ii++){memset(strtemp,0,sizeof(strtemp));if (ii==0) sprintf(strtemp,":%d",ii+1);if (ii >0) sprintf(strtemp,",:%d",ii+1);strcat(strInsertSQL,strtemp);strcat(strDeleteSQL,strtemp);}strcat(strInsertSQL,")");strcat(strDeleteSQL,")");sqlstatement insstmt(&conn);insstmt.prepare(strInsertSQL);sqlstatement delstmt(&conn);delstmt.prepare(strDeleteSQL);for (ii=0; ii<maxcount; ii++){insstmt.bindin(ii+1,strrowidn[ii],50);delstmt.bindin(ii+1,strrowidn[ii],50);}while (true) //每maxcount记录就执行一次{memset(strrowid,0,sizeof(strrowid));if (selstmt.next() != 0) break;strcpy(strrowidn[ccount],strrowid);ccount++;if (ccount == maxcount){if (insstmt.execute() != 0){if (insstmt.m_cda.rc != 1){logfile.Write("_hsmtables insert %s failed.\n%s\n",dsttname,insstmt.m_cda.message); return false;}}if (delstmt.execute() != 0){logfile.Write("_hsmtables delete %s failed.\n%s\n",dsttname,insstmt.m_cda.message); return false;}conn.commit();memset(strrowidn,0,sizeof(strrowidn));ccount=0;}if (fmod(selstmt.m_cda.rpc,10000) < 1){logfile.Write("%s to %s ok(%d).\n",srctname,dsttname,selstmt.m_cda.rpc); memset(localhour,0,sizeof(localhour)); // 判断当前时间是否在启动时间之内LocalTime(localhour,"hh24");if (strstr(hourstr,localhour)==0) return true;}}// 在以上循环处理的时候，如果不足maxcount，就在这里处理for (ii=0; ii<ccount; ii++){insstmt.prepare("\BEGIN\insert into %s(%s) select %s from %s where rowid=:1;\delete from %s where rowid=:2;\END;",dsttname,strColumnStr,strColumnStr,srctname,srctname);insstmt.bindin(1,strrowidn[ii],50);insstmt.bindin(2,strrowidn[ii],50);if (insstmt.execute() != 0){if (insstmt.m_cda.rc != 1){logfile.Write("_hsmtables insert %s or delete %s failed.\n%s\n",dsttname,srctname,insstmt.m_cda.message); return false;}}}conn.commit(); //一行一行的提交logfile.Write("%s to %s finish(%d).\n",srctname,dsttname,selstmt.m_cda.rpc);return true; }//1111111111111111111111111111111111获取dsttname表全部的列 bool GetColumnStr() {memset(strColumnStr,0,sizeof(strColumnStr)); char column_name[51];sqlstatement stmt(&conn);stmt.prepare("select lower(column_name) from USER_TAB_COLUMNS where table_name=upper('%s') order by column_id",dsttname);stmt.bindout(1,column_name,50);if (stmt.execute() != 0){logfile.Write("%s failed.\n%s\n",stmt.m_sql,stmt.m_cda.message); return false;}while(true){memset(column_name,0,sizeof(column_name));if (stmt.next()!=0) break;if (stmt.m_cda.rpc>1) strcat(strColumnStr,",");strcat(strColumnStr,column_name);}if (stmt.m_cda.rpc==0) { logfile.Write("表%s不存在。\n",dsttname); return false; }return true; }

$sqlplus shqx/pwdidc，如下数据都是一小时前生成，需要将后台生成数据脚本启动。

如下是数据迁移程序如何兼容其他数据库，oracle有rowid，mysql和pg都没有rowid但有keyid。

不管id字段是数字还是字符或字符串，我们都可用字符绑定。

如下不同数据库查数据字典得到全部列。

5.监控告警子系统：tmpfs，top，/proc/stat，/proc/meminfo，mount ，processes

我们处理后台数据也会有一个web系统用来管理参数，展示数据，查询数据等界面（不能让用户去登录plsqldeveloper查询）。如下是数据的监控，数据少于某数量显示红色。


如下日志等等都在sda(raid1，2个300G)中。

5.1 du，df，fdisk，lsblk：磁盘=硬盘

du：Disk Usage。-a：不仅统计目录还包括文件大小。-h：显示K/M/G等字节信息。-s：总的占用空间大小（-a是每个都统计）。一般文件夹大小4096b即4k（不包含里面文件大小）。


df：Disk Free。如果你想知道某个文件夹或文件大小用du，磁盘相关使用情况用df。

5.2 文件：建表入表，传输

以下是收集磁盘空间信息，现在插入一个U盘，fdisk -l可以看到U盘。



每个服务器上运行一个收集磁盘空间的小程序，收集到磁盘空间后生成xml文件存放在本地目录，通过文件传输系统或ftp将文件传给数据处理服务器（因为有的服务器不安装oracle客户端，只可安装文件传输客户端），统一保存到数据库。

//diskinfo.cpp，写入xml文件中再入库 #include "_public.h" void EXIT(int sig); CLogFile logfile;int main(int argc,char *argv[]) {if (argc != 4){printf("\n");printf("Using:./diskinfo hostname logfilename outputpath\n");printf("Example:/htidc/public/bin/diskinfo 118.89.50.198 /tmp/htidc/log/diskinfo.log /tmp/htidc/monclient\n\n");printf("此程序调用df命名，把本服务器的磁盘使用率信息写入xml文件。\n");printf("hostname是本服务器的主机名，为了方便识别，也可以用IP。\n");printf("logfilename是本程序的日志文件名。\n");printf("outputpath是输出的xml文件存放的目录。\n");printf("此程序运行在需要监控的服务器上（本程序只适用Linux系统），采集后的xml文件由文件传输程序发送给数据处理服务程序入库。\n\n\n");return -1;}CloseIOAndSignal(); signal(SIGINT,EXIT); signal(SIGTERM,EXIT);if (logfile.Open(argv[2],"a+") == false){printf("logfile.Open(%s) failed.\n",argv[2]); return -1;}FILE *fp=0;if ( (fp=popen("df -k --block-size=1M","r")) == NULL ){logfile.Write("popen(df -k --block-size=1M) failed.\n"); return false;}char strXMLFileName[301],strLocalTime[21];memset(strXMLFileName,0,sizeof(strXMLFileName));memset(strLocalTime,0,sizeof(strLocalTime));LocalTime(strLocalTime,"yyyymmddhh24miss");snprintf(strXMLFileName,300,"%s/diskinfo_%s_%s.xml",argv[3],strLocalTime,argv[1]);CFile XMLFile;if (XMLFile.OpenForRename(strXMLFileName,"w+") == false ){logfile.Write("XMLFile.OpenForRename(%s) failed.\n",strXMLFileName); pclose(fp); return -1;}XMLFile.Fprintf("<data>\n");CCmdStr CmdStr;char strBuffer[1024],strLine[500];while (true){memset(strBuffer,0,sizeof(strBuffer));if (FGETS(fp,strBuffer,500) == false) break;// 如果没有找到“%”，就再读取一行，与strBuffer拼起来if (strstr(strBuffer,"%") == 0){memset(strLine,0,sizeof(strLine));if (FGETS(fp,strLine,500) == false) break;strcat(strBuffer," "); strcat(strBuffer,strLine);}// 删除字符串前后的空格和换行符DeleteLRChar(strBuffer,' '); DeleteLRChar(strBuffer,'\n');// 把字符串中间的多个空格全部转换为一个空格UpdateStr(strBuffer," "," ");// 把全内容全部转换为小写ToLower(strBuffer);// 除了磁盘信息，还有可能是内存，SMB等其它文件，都丢弃掉if (strncmp(strBuffer,"/dev",4) != 0) continue;CmdStr.SplitToCmd(strBuffer," ");if (CmdStr.CmdCount() != 6) continue;char strusep[21];memset(strusep,0,sizeof(strusep));strcpy(strusep,CmdStr.m_vCmdStr[4].c_str());UpdateStr(strusep,"%","");char strLocalTime[21];memset(strLocalTime,0,sizeof(strLocalTime));LocalTime(strLocalTime,"yyyymmddhh24miss");XMLFile.Fprintf(\"<nodip>%s</nodip>"\"<crttime>%s</crttime>"\"<filesystem>%s</filesystem>"\"<total>%0.02f</total>"\"<used>%0.02f</used>"\"<available>%0.02f</available>"\"<usep>%0.02f</usep>"\"<mount>%s</mount><endl/>\n",argv[1],strLocalTime,CmdStr.m_vCmdStr[0].c_str(),atof(CmdStr.m_vCmdStr[1].c_str())/1024.0,atof(CmdStr.m_vCmdStr[2].c_str())/1024.0,atof(CmdStr.m_vCmdStr[3].c_str())/1024.0,(atof(CmdStr.m_vCmdStr[2].c_str())/atof(CmdStr.m_vCmdStr[1].c_str()))*100.0,CmdStr.m_vCmdStr[5].c_str());}XMLFile.Fprintf("</data>\n");pclose(fp);XMLFile.CloseAndRename();logfile.Write("create %s ok.\n",strXMLFileName);exit(0); }void EXIT(int sig) {if (sig > 0) signal(sig,SIG_IGN);logfile.Write("catching the signal(%d).\n",sig);logfile.Write("diskinfo exit.\n");exit(0); }

以下为收集CPU和内存信息，top命令显示如下，zombie表示僵尸，q退出，cpu用到20%算忙了。

ps -ef |grep htidc，cpuinfo.cpp思路是定义三个结构体变量，加载cpu信息到结构体里，睡60s，再继续加载cpu信息到结构体里，再将两结构体成员相减，就可以知道一分钟内cpu情况，采用的是一分钟信息。vi /proc/stat如下。

//cpuinfo.cpp #include "_public.h" void EXIT(int sig); CLogFile logfile; struct st_cpuinfo {double user;double sys;double wait;double nice;double idle;double irq;double softirq;double total; }; struct st_cpuinfo stcpuinfo1,stcpuinfo2,stcpuinfo3; bool LoadCPUInfo(struct st_cpuinfo &stcpuinfo);int main(int argc,char *argv[]) {if (argc != 4){printf("\n");printf("Using:./cpuinfo hostname logfilename outputpath\n");printf("Example:/htidc/public/bin/cpuinfo 118.89.50.198 /tmp/htidc/log/cpuinfo.log /tmp/htidc/monclient\n\n");printf("此程序读取/proc/stat文件，把本服务器的CPU使用率信息写入xml文件。\n");printf("hostname是本服务器的主机名，为了方便识别，也可以用IP。\n");printf("logfilename是本程序的日志文件名。\n");printf("outputpath是输出的xml文件存放的目录。\n");printf("此程序运行在需要监控的服务器上（本程序只适用Linux系统），采集后的xml文件由文件传输程序发送给数据处理服务程序入库。\n\n\n");return -1;}//memset(strHostName,0,sizeof(strHostName));//strncpy(strHostName,argv[2],20);CloseIOAndSignal(); signal(SIGINT,EXIT); signal(SIGTERM,EXIT);if (logfile.Open(argv[2],"a+") == false){printf("logfile.Open(%s) failed.\n",argv[2]); return -1;}memset(&stcpuinfo1,0,sizeof(struct st_cpuinfo));memset(&stcpuinfo2,0,sizeof(struct st_cpuinfo));memset(&stcpuinfo3,0,sizeof(struct st_cpuinfo));if (LoadCPUInfo(stcpuinfo1) ==false) return -1; sleep(60);if (LoadCPUInfo(stcpuinfo2) ==false) return -1;stcpuinfo3.user=stcpuinfo2.user-stcpuinfo1.user;stcpuinfo3.sys=stcpuinfo2.sys-stcpuinfo1.sys;stcpuinfo3.wait=stcpuinfo2.wait-stcpuinfo1.wait;stcpuinfo3.nice=stcpuinfo2.nice-stcpuinfo1.nice;stcpuinfo3.idle=stcpuinfo2.idle-stcpuinfo1.idle;stcpuinfo3.irq=stcpuinfo2.irq-stcpuinfo1.irq;stcpuinfo3.softirq=stcpuinfo2.softirq-stcpuinfo1.softirq;stcpuinfo3.total=stcpuinfo3.user+stcpuinfo3.sys+stcpuinfo3.wait+stcpuinfo3.nice+stcpuinfo3.idle+stcpuinfo3.irq+stcpuinfo3.softirq;char strLocalTime[21];memset(strLocalTime,0,sizeof(strLocalTime));LocalTime(strLocalTime,"yyyymmddhh24miss");char strXMLFileName[301];memset(strXMLFileName,0,sizeof(strXMLFileName));snprintf(strXMLFileName,300,"%s/cpuinfo_%s_%s.xml",argv[3],strLocalTime,argv[1]);CFile XMLFile;if (XMLFile.OpenForRename(strXMLFileName,"w+") == false ){logfile.Write("XMLFile.OpenForRename(%s) failed.\n",strXMLFileName); return -1;}XMLFile.Fprintf("<data>\n");XMLFile.Fprintf("<nodip>%s</nodip><crttime>%s</crttime><user>%0.02f</user><sys>%0.02f</sys><wait>%0.02f</wait><nice>%0.02f</nice><idle>%0.02f</idle><usep>%0.02f</usep><endl/>\n",argv[1],strLocalTime,stcpuinfo3.user/stcpuinfo3.total*100.0,stcpuinfo3.sys/stcpuinfo3.total*100.0,stcpuinfo3.wait/stcpuinfo3.total*100.0,stcpuinfo3.nice/stcpuinfo3.total*100.0,stcpuinfo3.idle/stcpuinfo3.total*100.0,100.0-stcpuinfo3.nice/stcpuinfo3.total*100.0);XMLFile.Fprintf("</data>\n");XMLFile.CloseAndRename();logfile.Write("create %s ok.\n",strXMLFileName);exit(0); } void EXIT(int sig) {if (sig > 0) signal(sig,SIG_IGN);logfile.Write("catching the signal(%d).\n",sig);logfile.Write("cpuinfo exit.\n");exit(0); }bool LoadCPUInfo(struct st_cpuinfo &stcpuinfo) {CFile CPUFile;if (CPUFile.Open("/proc/stat","r") == false ){logfile.Write("CPUFile.OpenForRead(/proc/stat) failed.\n"); return false;}CCmdStr CmdStr;char strBuffer[1024];while (true){memset(strBuffer,0,sizeof(strBuffer));if (CPUFile.FFGETS(strBuffer,500) == false) break;// 删除字符串前后的空格DeleteLRChar(strBuffer,' ');// 把字符串中间的多个空格全部转换为一个空格UpdateStr(strBuffer," "," ");ToLower(strBuffer);CmdStr.SplitToCmd(strBuffer," ");if (strcmp(CmdStr.m_vCmdStr[0].c_str(),"cpu")==0) {stcpuinfo.user=atof(CmdStr.m_vCmdStr[1].c_str());stcpuinfo.sys=atof(CmdStr.m_vCmdStr[2].c_str());stcpuinfo.wait=atof(CmdStr.m_vCmdStr[3].c_str());stcpuinfo.nice=atof(CmdStr.m_vCmdStr[4].c_str());stcpuinfo.idle=atof(CmdStr.m_vCmdStr[5].c_str());stcpuinfo.irq=atof(CmdStr.m_vCmdStr[6].c_str());stcpuinfo.softirq=atof(CmdStr.m_vCmdStr[7].c_str());return true;}}logfile.Write("Read /proc/stat failed.\n"); return false; }

vi /tmp/htdic/monclient/cpu*，如下是收集到的信息。

收集内存信息#free -m，和top命令查看的内存是一样的，也在系统文件vi /proc/meminfo。

以下是收集Oracle表空间信息，表空间就像磁盘空间一样，表空间的信息收集要去读取oracle的数据字典，如下sql是查询oracle表空间使用率，取出后写入xml文件里，vi tbspaceinfo.cpp，要dba权限。

select * from ( Select a.tablespace_name, to_char(a.bytes/1024/1024,'99,999.999') total_bytes, to_char(b.bytes/1024/1024,'99,999.999') free_bytes, to_char(a.bytes/1024/1024 - b.bytes/1024/1024,'99,999.999') use_bytes, to_char((1 - b.bytes/a.bytes)*100,'99.99') || '%' use from (select tablespace_name, sum(bytes) bytes from dba_data_files group by tablespace_name) a, (select tablespace_name, sum(bytes) bytes from dba_free_space group by tablespace_name) b where a.tablespace_name = b.tablespace_name union all select c.tablespace_name, to_char(c.bytes/1024/1024,'99,999.999') total_bytes, to_char( (c.bytes-d.bytes_used)/1024/1024,'99,999.999') free_bytes, to_char(d.bytes_used/1024/1024,'99,999.999') use_bytes, to_char(d.bytes_used*100/c.bytes,'99.99') || '%' use from (select tablespace_name,sum(bytes) bytes from dba_temp_files group by tablespace_name) c, (select tablespace_name,sum(bytes_cached) bytes_used from v$temp_extent_pool group by tablespace_name) d where c.tablespace_name = d.tablespace_name )

//tbspaceinfo.cpp #include "_public.h" #include "_ooci.h" void EXIT(int sig); struct st_TBSPACEINFO {long taskid;char nodip[31];char tablespace[101];double total;double used;double available;double usep;double alarmvalue;int alarmsts;char crttime[21];int rsts; }; struct st_TBSPACEINFO stTBSPACEINFO; CLogFile logfile; connection conn; int main(int argc,char *argv[]) {if (argc != 5){printf("\n");printf("Using:./tbspaceinfo hostname logfilename outputpath username/password@tnsnames\n");printf("Example:/htidc/public/bin/tbspaceinfo 10.153.98.13 /tmp/htidc/log/tbspaceinfo_10.153.98.13.log /tmp/htidc/monclient shqx/pwdidc@SZQX_10.153.98.13\n\n");printf("此程序连接远程数据库，把远程数据库表空间使用率信息写入xml文件。\n");printf("hostname是本服务器的主机名，为了方便识别，也可以用IP。\n");printf("logfilename是本程序的日志文件名。\n");printf("outputpath是输出的xml文件存放的目录。\n");printf("username/password@tnsnames为待监控的远程数据库的用户名/密码@连接名。\n");printf("此程序运行在数据中心应用程序的服务器上。\n\n\n");return -1;}// 关闭全部的信号和输入输出// 设置信号,在shell状态下可用 "kill + 进程号" 正常终止些进程// 但请不要用 "kill -9 +进程号" 强行终止CloseIOAndSignal(); signal(SIGINT,EXIT); signal(SIGTERM,EXIT);if (logfile.Open(argv[2],"a+") == false){printf("logfile.Open(%s) failed.\n",argv[2]); return -1;}if (conn.connecttodb(argv[4],"Simplified Chinese_China.ZHS16GBK") != 0){logfile.Write("conn.connecttodb(%s) failed.\n",argv[4]); return -1;}sqlstatement stmt;stmt.connect(&conn);stmt.prepare("\select f.tablespace_name,a.total,u.used,f.free,(u.used/a.total)*100 from\(select tablespace_name,sum(bytes/(1024*1024*1024)) total from DBA_DATA_FILES\group by tablespace_name) a,\(select tablespace_name,round(sum(bytes/(1024*1024*1024))) used from DBA_EXTENTS\group by tablespace_name) u,\(select tablespace_name,round(sum(bytes/(1024*1024*1024))) free from DBA_FREE_SPACE\group by tablespace_name) f\where a.tablespace_name = f.tablespace_name\and a.tablespace_name = u.tablespace_name\and f.tablespace_name in (select tablespace_name from DBA_TABLESPACES where contents='PERMANENT')");stmt.bindout(1,stTBSPACEINFO.tablespace,100);stmt.bindout(2,&stTBSPACEINFO.total);stmt.bindout(3,&stTBSPACEINFO.used);stmt.bindout(4,&stTBSPACEINFO.available);stmt.bindout(5,&stTBSPACEINFO.usep);if (stmt.execute() != 0){logfile.Write("select DBA_DATA_FILES,DBA_EXTENTS,DBA_FREE_SPACE failed.\n%s\n",stmt.m_cda.message); return -1;}char strLocalTime[21];memset(strLocalTime,0,sizeof(strLocalTime));LocalTime(strLocalTime,"yyyymmddhh24miss");char strXMLFileName[301];memset(strXMLFileName,0,sizeof(strXMLFileName));snprintf(strXMLFileName,300,"%s/tbspaceinfo_%s_%s.xml",argv[3],strLocalTime,argv[1]);CFile XMLFile;if (XMLFile.OpenForRename(strXMLFileName,"w+") == false ){logfile.Write("XMLFile.OpenForRename(%s) failed.\n",strXMLFileName); return -1;}XMLFile.Fprintf("<data>\n");while (true){memset(&stTBSPACEINFO,0,sizeof(stTBSPACEINFO));if (stmt.next() != 0) break;XMLFile.Fprintf(\"<nodip>%s</nodip>"\"<crttime>%s</crttime>"\"<tablespace>%s</tablespace>"\"<total>%0.02f</total>"\"<used>%0.02f</used>"\"<available>%0.02f</available>"\"<usep>%0.02f</usep><endl/>\n",argv[1],strLocalTime,stTBSPACEINFO.tablespace,stTBSPACEINFO.total,stTBSPACEINFO.used,stTBSPACEINFO.available,stTBSPACEINFO.usep);}XMLFile.Fprintf("</data>\n");XMLFile.CloseAndRename();logfile.Write("生成文件%s.\n",strXMLFileName);exit(0); } void EXIT(int sig) {if (sig > 0) signal(sig,SIG_IGN);logfile.Write("catching the signal(%d).\n",sig);logfile.Write("tbspaceinfo exit.\n");exit(0); }

以下是收集Oracle会话信息，我们用客户端通oracle的监听连上oracle数据库，oracle数据库会启动一个进程向会话提供服务，是多进程的服务端，每增加一连接(进程)，需要消耗系统资源(内存，socket连接)。如下一共有6个客户端连上，SQL>exit退出一个就剩下5个。ps -ef |grep oracle可查看oracle数据库系统进程。


如下两个sql是查询数据字典查到进程和上面一样，第一个sql是查全部包括系统进程。如下表有一个OSUSER表示连上来的客户端操作系统名字，还可以通过SQL_ID查到连上来的客户端sql语句操作。还可以指定oracle用户的会话数。

// dbsessioninfo.cpp #include "qxmon.h" void EXIT(int sig); struct st_DBSESSIONINFO stDBSESSIONINFO; CLogFile logfile; connection conndst; CProgramActive ProgramActive;int main(int argc,char *argv[]) {if (argc != 5){printf("\n");printf("Using:./dbsessioninfo hostname logfilename outputpath username/password@tnsnames\n");printf("Example:/htidc/htidc/bin/procctl 300 /htidc/qxmon/bin/dbsessioninfo 10.153.98.13 /log/qxmon/dbsessioninfo_10.153.98.13.log /qxdata/qxmon/qxmonclient szidc/pwdidc@SZQX_10.153.98.13\n\n");printf("此程序连接远程数据库，把远程数据库会话信息写入XML文件。\n");printf("hostname是本服务器的主机名，为了方便识别，也可以用IP。\n");printf("logfilename是本程序的日志文件名。\n");printf("outputpath是输出的XML文件存放的目录。\n");printf("username/password@tnsnames为待监控的远程数据库的用户名/密码@连接名。\n");printf("此程序运行在监控平台的服务器上。\n\n\n");return -1;}CloseIOAndSignal(); signal(SIGINT,EXIT); signal(SIGTERM,EXIT);if (logfile.Open(argv[2],"a+") == FALSE){printf("logfile.Open(%s) failed.\n",argv[2]); return -1;}ProgramActive.SetProgramInfo(&logfile,"alarmserver",300);if (conndst.connecttodb(argv[4])!=0){logfile.Write("connmon.connecttodb(%s) failed.\n",argv[4]); EXIT(-1);}sqlstatement stmt;stmt.connect(&conndst);stmt.prepare("select count(*) from V$SESSION where type!='BACKGROUND' and status!='KILLED'"); stmt.bindout(1,&stDBSESSIONINFO.total);if (stmt.execute() != 0){logfile.Write("select V$SESSION failed.\n%s\n",stmt.cda.message); EXIT(-1);}char strLocalTime[21];memset(strLocalTime,0,sizeof(strLocalTime));LocalTime(strLocalTime,"yyyymmddhh24miss");char strXMLFileName[301];memset(strXMLFileName,0,sizeof(strXMLFileName));snprintf(strXMLFileName,300,"%s/dbsessioninfo_%s_%s.xml",argv[3],strLocalTime,argv[1]);CFile XMLFile;if (XMLFile.OpenForRename(strXMLFileName,"w+") == FALSE ){logfile.Write("XMLFile.OpenForRename(%s) failed.\n",strXMLFileName); EXIT(-1);}XMLFile.Fprintf("<data>\n");while (TRUE){memset(&stDBSESSIONINFO,0,sizeof(stDBSESSIONINFO));if (stmt.next() != 0) break;XMLFile.Fprintf(\"<nodip>%s</nodip>"\"<crttime>%s</crttime>"\"<total>%d</total><endl/>\n",argv[1],strLocalTime,stDBSESSIONINFO.total);}XMLFile.Fprintf("</data>\n");XMLFile.CloseAndRename();logfile.Write("生成文件%s.\n",strXMLFileName);exit(0); }void EXIT(int sig) {if (sig > 0) signal(sig,SIG_IGN);logfile.Write("catching the signal(%d).\n",sig);logfile.Write("dbsessioninfo exit.\n");exit(0); }

oracle里有一个系统参数processes（最大允许的会话总数）。_ooci.h中connection类连接上一个后进程及processes+1，sqlstatement类实例化并指定数据库连接也会消耗一种资源即open_cursors参数。oracle有缓存机制存sql，下次用到则不进行语法分析，所以oracle需大内存，存的越多，缓存命中率越高。

总结

以上是生活随笔为你收集整理的【C/C++13】天气APP：数据挖掘/HTTP协议/非结构化数据存储（filetoblob.cpp），数据管理/监控告警（hsmtable.cpp，tbspaceinfo.cpp）的全部内容，希望文章能够帮你解决所遇到的问题。

如果觉得生活随笔网站内容还不错，欢迎将生活随笔推荐给好友。