日志系统的设计与实现

日志系统的设计与实现

Scroll Down

需求

用户的行为操作、物品消耗产出需要记录,项目的日志打印底层调用so文件,无法修改,于是决定自己写一套脚本实现日志系统
大概流程为:打印日志到log文件——脚本定时提取文件——根据信息输出日志名称和参数到内网共享文件——加载进日志数据库——运营管理web平台查询
目前效果如下
参数通过[$@$]分隔,[~*$]终止符
image.png

配置信息

根据filename输出日志名称,idnamelist参数列表

local server = require "server"
local OSSLogConfig = {}

OSSLogConfig.FileType = {
	ItemDB				= 1,	-- 物品日志记录
	ActorLevelDB		= 2,	-- 角色升级日志记录
}

OSSLogConfig.TypeName = {
	[OSSLogConfig.FileType.ItemDB] = {
		filename = "ItemDB",
		--[[物品日志:playerId:用户playerdbid,serverid:服务器id,account:账号id,itemId:物品itemid,itemdDbid:物品唯一dbid,count:数量,moduleType:功能模块type产出,对应BaseConfig.YuanbaoRecordType
		addType:增加类型,默认增加,1是减少,attrs:物品属性,nowtime:当前时间]]
		idnamelist = {"playerId","serverid","account","itemid","itemdDbid","count","addType","moduleType","itemType","attrs","nowtime"},
	},
	[OSSLogConfig.FileType.ActorLevelDB] = {
		filename = "ActorLevelDB",
		--[[物品日志:playerId:用户playerdbid,serverid:服务器id,level:玩家等级,nowtime:当前时间]]
		idnamelist = {"playerId","serverid","account","level","nowtime"},
	},
}

return OSSLogConfig

项目日志代码

打印日志

	--升级日志
	local logdate = {level = level}
	lua_app.log_info(lua_util.GetOSSLogInfo(OSSLogConfig.FileType.ActorLevelDB,logdate,self))
	server.playerCenter:onevent(server.event.levelup, self, oldlevel, level)

	--经验日志
	local logdate = {itemid = ItemConfig.NumericType.Exp, count = count}
	lua_app.log_info(lua_util.GetOSSLogInfo(OSSLogConfig.FileType.ItemDB,logdate,self))

	--元宝日志
	local logdate = {itemid = ItemConfig.NumericType.YuanBao,count = count,moduleType = type}
	lua_app.log_info(lua_util.GetOSSLogInfo(OSSLogConfig.FileType.ItemDB,logdate,self))

工具类获取日志字符串

--获取日志字符串#define  [@logfile] 文件名和ip
--OSS_LOG_VALUE_SPACE		"[$@$]"
--#define OSS_LOG_ENDLINE			"[~*$]"
function lua_util.GetOSSLogInfo(filetype,logdata,player)
	local logcfg = OSSLogConfig.TypeName[filetype]
	local logstr = "[@logfile]"..logcfg.filename.."[@logfile]"
	if player then
		logdata.playerId = player.dbid
		logdata.serverid = player.cache.serverid
		logdata.account = player.cache.account
	end
	--默认增加
	if logdata.count and logdata.count < 0 then
		logdata.addType = 1
	else
		logdata.addType = 0
	end
	for _,name in pairs(logcfg.idnamelist) do
		--日志时间和结束符
		if not name == "nowtime" then
			if logdata[name] then
				logstr = logstr..logdata[name]
			end
			logstr = logstr.."[$@$]"
		else
			logstr = logstr..lua_app.now().."[~*$]"
		end
	end
	return logstr
end

Python定时脚本

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import os,sys,io,importlib,time,chardet
sys.setrecursionlimit(10000000)
importlib.reload(sys)
init_flag = True  #初次加载程序
time_kick = 60
record_count = 0
record_file = time.strftime('%Y-%m-%d-%H', time.localtime())
final = []
isLoadAll = input("True/False True:加载历史日志文件,False:只定时刷新")
resultwork = os.path.dirname(__file__)+"/" #目标路径
"""
安装pysvn 
def get_login(realm, username, may_save):
    retcode = True    #True,如果需要验证;否则用False
    username = 'liujun'    #用户名
    password = '49vO9L7Z'    #密码
    save = False    #True,如果想之后都不用验证;否则用False
    return retcode, username, password, save
client = pysvn.Client()
client.callback_get_login = get_login

#svnurl='https://172.18.0.160:8445/svn/GoodGame/ProgramH5Server/H5文档/h5log'
svnurl=''

client.checkout(svnurl, resultwork)    #检出最新版本
"""

#work为绝对路径,windos下"d:/ProgramH5Server/H5_server_lj/log"
#linuxs下wolk = "/H5_server/log"  "/svn/H5_server_lj/log"
#linux运行1:chmod a+x log.py  2:sed -i 's/\r$//' log.py 3.sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 150 4:./log.py 
work = "/H5_server/log"   #提取目标日志文件
for root,dirs,files in os.walk(work): 
    for file in files: 
        final.append(os.path.join(root,file))    


#定义获取文件编码的函数
def get_encoding(file):
    with open(file,'rb') as f:
        return chardet.detect(f.read())['encoding']

def writeline(file,record_count,isRecord):
    log_dir = os.path.join(work, file)
    operate_line = [[]]
    logfllelist = []
    #fileEncoding = get_encoding(file)
    try:
        with io.open(log_dir, 'r', encoding='utf-8') as f: # 'utf-8'
            start = record_count
            for line in f.readlines()[start:]:  
                if isRecord:
                    record_count = record_count + 1      
                if '[@logfile]' in line :
                    strlist = line.split('[@logfile]')
                    logflle = strlist[1] #对应OSSLogConfig.TypeName.filename
                    if logflle not in logfllelist:
                        logfllelist.append(logflle) #对应OSSLogConfig.TypeName.idnamelist
                    operate_line[logfllelist.index(logflle)].append(strlist[2])
    except:
        pass      
    f.close()

    for logflle in logfllelist:
        try:
            log_b = os.path.join(resultwork, logflle+".log")
            with io.open(log_b, 'a', encoding='utf-8') as result:
                for i in list(set(operate_line[logfllelist.index(logflle)])):
                    result.write(i)
            #client.checkin([resultwork+logflle+".log"], '日志更新')
        except:
            pass  

if isLoadAll == 'True':
    for file in final:
        writeline(file,0,False)


#定时 从最近文件开始扫描记录
while True:
    try:
        filedata = file_name = time.strftime('%Y-%m-%d', time.localtime())
        file_name = time.strftime('%Y-%m-%d-%H', time.localtime())
        #2020-09-22-20.log
        file = os.path.join(work, filedata + "\\" + record_file +".log")
        if init_flag:
            with io.open(file,'r',encoding='utf-8') as f: 
                for line in f.readlines():
                    record_count = record_count + 1
            f.close()
            init_flag = False
            pass
        #如果文件更新,则更新数据并重置行数
        #client.update(resultwork)
        if record_file != file_name:
            #重新开始读新文件,旧文件全部读取
            writeline(file,record_count,True)
            record_count = 0    
            record_file = file_name
            file = os.path.join(work, filedata + "\\" + record_file +".log")
        writeline(file,record_count,True)
    except:
        pass
    time.sleep(time_kick)

提取错误日志文件脚本:
未完成,暂时版本

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import os,sys,io,importlib,time,chardet
sys.setrecursionlimit(10000000)
importlib.reload(sys)
init_flag = True  #初次加载程序
time_kick = 10
record_count = 0
record_file = time.strftime('%Y-%m-%d-%H', time.localtime())
final = []
isLoadAll = input("True/False True:加载历史日志文件,False:只定时刷新")
resultwork = os.path.dirname(__file__)+"/" #目标路径

#work为绝对路径,windos下"d:/ProgramH5Server/H5_server_lj/log"
#linuxs下wolk = "/H5_server/log"  "/svn/H5_server_lj/log"
#linux运行1:chmod a+x log.py  2:sed -i 's/\r$//' log.py 3.sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 150 4:./log.py 
work = "/H5_server/log"   #提取目标日志文件
for root,dirs,files in os.walk(work): 
    for file in files: 
        final.append(os.path.join(root,file))    

def writeline(file,record_count,isRecord):
    log_dir = os.path.join(work, file)
    error_line = []
    try:
        with io.open(log_dir, 'r', encoding='utf-8') as f: # 'utf-8'
            start = record_count
            for line in f.readlines()[start:]:  
                if isRecord:
                    record_count = record_count + 1      
                if 'ERROR' in line :
                    error_line.append(line)                             
    except:
        pass      
    f.close()
    errorlogname = 'error.log'
    resultwork = os.path.dirname(__file__)
    log_a = os.path.join(resultwork, errorlogname)
    try:
        log_b = os.path.join(resultwork, logflle+".log")
        with io.open(log_a, 'a',encoding='utf-8') as result:
        for i in list(set(error_line)):
            result.write(i)
    except:
        pass  
   
if isLoadAll == 'True':
    for file in final:
        writeline(file,0,False)


#定时 从最近文件开始扫描记录
while True:
    try:
        filedata = file_name = time.strftime('%Y-%m-%d', time.localtime())
        file_name = time.strftime('%Y-%m-%d-%H', time.localtime())
        #2020-09-22-20.log
        file = os.path.join(work, filedata + "\\" + record_file +".log")
        if init_flag:
            with io.open(file,'r',encoding='utf-8') as f: 
                for line in f.readlines():
                    record_count = record_count + 1
            f.close()
            init_flag = False
            pass
        #如果文件更新,则更新数据并重置行数
        #client.update(resultwork)
        if record_file != file_name:
            #重新开始读新文件,旧文件全部读取
            writeline(file,record_count,True)
            record_count = 0    
            record_file = file_name
            file = os.path.join(work, filedata + "\\" + record_file +".log")
        writeline(file,record_count,True)
    except:
        pass
    time.sleep(time_kick)