#!/usr/bin/env python
###################################################
#Teach wisedom to my machine,please call me Croco#
# 定時更改/etc/hosts 實現不修改程序的情況下，使得程序
# curl url 自動解析到內網IP
##################################################
from DebugLogger import *
Logger.init("updatehost")
import os
import sys
import time
import json
import random
special_host = "chelun.eclicks.cn"
last_data = ""
last_line=""
def selectAnotherIp(iplist,last_ip):
if last_ip in iplist:
iplist.remove(last_ip)
if len(iplist) == 0:
return ""
return random.choice(iplist)
def readWebConfAndGetNewIp(last_ip):
file_name = "web.conf"
if not os.path.exists(file_name):
log("not exits such file:{0}".format(file_name))
sys.exit(0)
return ""
old_data = ""
with open(file_name, "r") as f:
old_data = f.read()
if not old_data:
log("get empty file:{0}".format(file_name))
return ""
iplist = []
try:
js_obj = json.loads(old_data)
for item in js_obj["servers"]:
if item["status"] == "online":
iplist.append(item["host"])
except:
log("read failed from webserver_conf:" + file_name + "data:" + old_data)
return ""
return selectAnotherIp(iplist,last_ip)
def task():
global last_data
global special_host
file_name = "hosts.conf"
file_bak_name ="hosts2.conf"
if not os.path.exists(file_name):
log("not exits such file:{0}".format(file_name))
sys.exit(0)
return
old_data = ""
with open(file_name,"r") as f:
old_data = f.readlines()
if not old_data:
log("get empty file:{0}".format(file_name))
return
if last_data == old_data:
return
line_list1=[]
line_list2 = []
for line in old_data:
line = line.strip()
if not line:
continue
if "#" in line:
line_list1.append(line)
line_list2.append(line)
continue
if special_host not in line:
line_list1.append(line)
line_list2.append(line)
continue
line_list1.append(line)
arr= line.split(" ")
if len(arr) != 2:
log("error line:{0}".format(line))
sys.exit(1)
return
ipaddr = arr[0]
newipaddr = readWebConfAndGetNewIp(ipaddr)
if not newipaddr:
log("not need to change")
return
line = newipaddr+" "+special_host
line_list2.append(line)
log("change ipaddr:{0} -> newipaddr:{1}".format(ipaddr,newipaddr))
if os.path.exists(file_bak_name):
os.remove(file_bak_name)
with open(file_bak_name,"w") as f:
for line in line_list1:
f.write(line)
f.write("\n")
os.remove(file_name)
with open(file_name,"w") as f:
for line in line_list2:
f.write(line)
f.write("\n")
if not os.path.exists(file_name):
log("update failed,file:{0}".format(file_name),2)
return
log("update succ,file:{0}".format(file_name))
with open(file_name,"r") as f:
last_data = f.readlines()
pass
def main():
uid = os.getuid()
if uid != 0:
log("os.getuid() != 0,current uid:{0}".format(uid))
return
while True:
task()
time.sleep(60)
pass
if __name__ == '__main__':
task()
#main()
]]>
#HOW TO MAKE PASSWORD
#BY ZHANGTAO
import random,sys,time
def main():
if len(sys.argv) != 2:
print "usage:python {0} length_of_password".format(sys.argv[0])
return
c=sys.argv[1]
c=int(c)
a1="123456789"
a2="qwertyuipasdfghjklzxcvbnm"
a3="QWERTYUIOPADFGHJKLZXCBNM"
a4="@$&"
aList=[a1,a2,a3,a4]
s=""
aLast=5
for i in xrange(c):
while True:
a = random.choice(aList)
if len(a) != aLast:
aLast=len(a)
if aLast == len(a4):
aList = [a1, a2, a3]
break
time.sleep(0.1)
s += random.choice(a)
print "make password:",s
print "len :",len(s)
if __name__=="__main__":
main()
]]>
# encoding:utf-8
# zhangtao 2016/07-28
# monit the change of file;
#if file is modified,send httpGet request to service,then service reload config
#############################################
import os
import urllib2
from urllib2 import URLError, HTTPError
from pyinotify import WatchManager, Notifier, ProcessEvent,IN_DELETE, IN_CREATE,IN_MODIFY
import threading
from debuglogger import *
config_urls=["http://127.0.0.1:6001/chelun?query=reload1","http://127.0.0.1:6001/chelun?query=reload2","http://127.0.0.1:6001/chelun?query=reload3"]
config_files=["service:dispatch.conf","service:proxy.conf","service:qc.conf"]
config_flags=[0,0,0]
config_timer=0
pid_file="run/daemon.pid"
def doGet(url):
request = urllib2.Request(url)
try:
response = urllib2.urlopen(request, timeout=10)
page = response.read()
return page
except URLError, e:
if hasattr(e, 'code'):
s='The server couldn\'t fulfill the request. errorcode:{0} url:{1}'.format(e.code,url)
return s
elif hasattr(e, 'reason'):
s='We failed to reach a server. reason:{0} url:{1}'.format(e.reason,url)
return s
return "Error"
def func():
global config_files
global config_flags
global config_urls
global config_timer
config_timer=0
for i in xrange(3):
if 1==config_flags[i]:
result=doGet(config_urls[i])
config_flags[i]=0
s="fileChange:{0} triggerUrl:{1} rspResult:{2}".format( config_files[i],config_urls[i],result )
Logger.dumplog(s)
def startTimer():
global config_timer
if config_timer != 0:
return
config_timer=1
timer = threading.Timer(5, func)
timer.start()
class EventHandler(ProcessEvent):
"""事件處理"""
def process_IN_CREATE(self, event):
#print "Create file: %s " % os.path.join(event.path,event.name)
pass
def process_IN_DELETE(self, event):
#print "Delete file: %s " % os.path.join(event.path,event.name)
pass
def process_IN_MODIFY(self, event):
#print "Modify file: %s " % os.path.join(event.path,event.name)
pass
if "service:dispatch.conf" == event.name:
config_flags[0] = 1
startTimer()
elif "service:proxy.conf" == event.name:
config_flags[1] = 1
startTimer()
elif "service:qc.conf" == event.name:
config_flags[2] = 1
startTimer()
def FSMonitor(path='.'):
wm = WatchManager()
#mask = IN_DELETE | IN_CREATE |IN_MODIFY
mask = IN_MODIFY
notifier = Notifier(wm, EventHandler())
wm.add_watch(path, mask,auto_add=True,rec=True)
s= 'now starting monitor %s'%(path)
Logger.dumplog(s)
pid=str( os.getpid() )
with open(pid_file,"w") as f:
f.write( pid )
s="pid:{0}".format(pid)
Logger.dumplog( s)
while True:
try:
notifier.process_events()
if notifier.check_events():
notifier.read_events()
except KeyboardInterrupt:
notifier.stop()
break
if __name__ == "__main__":
FSMonitor('/data/config/')
]]>
# http://blog.csdn.net/zcyhappy1314/article/details/8283717
import MySQLdb
import sys
import os
import time
mysql = r"/usr/bin/mysql"
mysqldump = r"/usr/bin/mysqldump"
mysql_host = "127.0.0.1"
mysql_port = 3306
mysql_user = "root"
mysql_pwd = ""
save_dir = "temp"
def mkdir(path):
path = path.strip()
path = path.rstrip("\\")
if not os.path.exists(path):
os.makedirs(path)
return path
def pause():
raw_input("please enter to continue!")
print "doing.."
# dump database ,only db-struct,
def dumpDatabaseOnlyStruct(dbName="",dbCount=0):
print "dumpDatabaseOnlyStruct:",dbName,dbCount
databases=""
if dbCount==0:
databases=dbName
else:
for i in xrange(dbCount):
databases += "{0}_{1} ".format(dbName,i)
dbFile = r"{0}/{1}.sql".format(save_dir, dbName)
if os.path.exists(dbFile):
print "remove file:", dbFile
os.remove(dbFile)
########################
cmdStr = "{0} -h{1} -u{2} -p{3} -d -B {4} > {5}".format(mysqldump, mysql_host, mysql_user, mysql_pwd, databases,dbFile)
print "------------------dumpDatabaseOnlyStruct---------------"
print cmdStr
pause()
t1 = time.time()
result = os.system(cmdStr)
t2 = time.time()
t3 = t2 - t1
print "dumpDatabaseOnlyStruct cmd:{0} \n>>result:{1} useTime:{2} file:{3}".format(cmdStr, result, t3, dbFile)
# dump database , db-struct,+ db-data
#db_im_msg_record
def dumpDatabaseStructAndData(dbName="",dbCount=0):
databases=""
if dbCount==0:
databases=dbName
else:
for i in xrange(dbCount):
databases += "{0}_{1} ".format(dbName,i)
dbFile = r"{0}/{1}.sql".format(save_dir, dbName)
if os.path.exists(dbFile):
print "remove file:", dbFile
os.remove(dbFile)
########################
cmdStr = "{0} -h{1} -u{2} -p{3} -B {4} > {5}".format(mysqldump, mysql_host, mysql_user, mysql_pwd, databases, dbFile)
print "------------------dumpDatabaseStructAndData---------------"
print cmdStr
pause()
t1 = time.time()
result = os.system(cmdStr)
t2 = time.time()
t3 = t2 - t1
print "dumpDatabaseOnlyStruct cmd:{0} \n>>result:{1} useTime:{2} file:{3}".format(cmdStr, result, t3, dbFile)
# recover from mysql_dump_sql_file
def recoverMysqlFromDumpfile(dbName):
dbFile = r"{0}/{1}.sql".format(save_dir, dbName)
if not os.path.exists(dbFile):
print "recoverMysqlFromDumpfile,but not find file:{0}".format(dbFile)
sys.exit(-1)
cmdStr = '{0} -h{1} -u{2} -p{3} -e "source {4}"'.format(mysql, mysql_host, mysql_user, mysql_pwd, dbFile)
print "------------------recoverMysqlFromDumpfile---------------"
print cmdStr
pause()
t1 = time.time()
result = os.system(cmdStr)
t2 = time.time()
t3 = t2 - t1
print "recoverMysqlFromDumpfile cmd:{0} result:{1} useTime:{2}".format(cmdStr, result, t3)
def dumpMysqlDB():
dumpDatabaseOnlyStruct("db_im_msg_record",8)
dumpDatabaseOnlyStruct("db_im_user_msg", 4)
dumpDatabaseOnlyStruct("db_im_group", 8)
#dumpDatabaseStructAndData("db_im_msg_record",8)
def recoverMysqlDB():
recoverMysqlFromDumpfile("db_im_msg_record")
recoverMysqlFromDumpfile("db_im_user_msg")
recoverMysqlFromDumpfile("db_im_group")
if __name__ == "__main__":
print "-------begin--------"
mkdir(save_dir)
dumpMysqlDB()
# recoverMysqlDB()
print "-------done---------"
]]>
#Give wisdom to the machine,By ChangShouMeng\
#定義傳輸的上下文，client與server通用的上下文
import time
import os,sys,traceback
class FileTransferContext(object):
TRANSFER_TYPE_SEND=0
TRANSFER_TYPE_RECV=1
def __init__(self):
self.transferType=0
self.filePath = "" # absolute path,send
self.fileTotalSize = 0
self.fileChunkSize=1024
self.contextName="-"
self.fileBaseName=""#only filename,recv
self.fileHandle = None
self.transferedSize=0
self.transferMsgId = 0
self.transferSeqNum = 0
self.transferSeqNumUpLimit = 0
self.transferSeqChunkSize = 10
self.transferBeginTime = 0
self.tmpdir="temp"
self.workStatus=0
def __del__(self):
self.finiTransInfo()
def initTransferInfo(self):
print "---begin----init transfer info"
try:
if self.transferType== FileTransferContext.TRANSFER_TYPE_SEND:
self.contextName="snd>>"
self.fileTotalSize = os.path.getsize(self.filePath)
self.fileBaseName = os.path.basename(self.filePath)
self.fileHandle=open(self.filePath,'rb')
else:
self.contextName="rcv<<"
tmp_fn = r"{0}/{1}_tmp".format(self.tmpdir, self.fileBaseName)
self.fileHandle=open(tmp_fn,'wb')
print tmp_fn
assert self.fileHandle
self.transferBeginTime= (int)(time.time())
self.transferedSize=0
self.transferSeqNumUpLimit=self.fileTotalSize/self.fileChunkSize + 1
self.workStatus=1
print "initTransferInfo {0} fileName:{1} fileSize:{2}".format(self.contextName,self.fileBaseName,self.fileTotalSize)
return True
except:
print "initTransferinfo failed:",traceback.print_exc()
print "init filepath is:",self.filePath
return False
def finiTransInfo(self):
if not self.fileHandle:
return
try:
if self.transferType == FileTransferContext.TRANSFER_TYPE_RECV:
self.fileHandle.flush()
self.fileHandle.close()
self.fileHandle=None
except:
print "finiTransInfo failed:", traceback.print_exc()
def traceTransferInfo(self):
if self.transferedSize == self.fileTotalSize:
self.workStatus=2
if self.transferSeqNum > 5 and (self.transferSeqNum % self.transferSeqChunkSize) != 0:
return
useTime = int(time.time()) - self.transferBeginTime
speed = ""
progress = float(self.transferedSize) / float(self.fileTotalSize)
if useTime > 0:
speedint = float(self.transferedSize) / float(useTime * 1024)
if speedint >= 1024:
speedint= float(speedint)/ float(1024)
speed="{:.2f}mb/s".format(speedint)
else:
speed="{:.2f}kb/s".format(speedint)
pass
print "{0}msgid:{1} total:{2}byte tranfered:{3}byte speed:{4} progress:{5:.2f} use-time:{6}s".format(
self.contextName,
self.transferMsgId,
self.fileTotalSize,
self.transferedSize,
speed,
progress,
useTime)
def readBuffer(self):
try:
return self.fileHandle.read(self.fileChunkSize)
except:
print "readBuffer err:",traceback.print_exc()
return ""
def writeBuffer(self,buffer):
try:
self.fileHandle.write(buffer)
if (self.transferSeqNum % self.transferSeqChunkSize) == 0:
self.fileHandle.flush()
self.transferedSize += len(buffer)
#print self.transferedSize,self.fileTotalSize
if self.transferedSize == self.fileTotalSize:
self.fileHandle.flush()
self.fileHandle.close()
self.fileHandle=None
tmp_fn = "{0}/{1}_tmp".format(self.tmpdir, self.fileBaseName)
real_fn = "{0}/{1}".format(self.tmpdir, self.fileBaseName)
print "-"*80
print "tmp_fn:",tmp_fn
print "real_fn:",real_fn
if os.path.exists(real_fn):
os.remove(real_fn)
os.rename(tmp_fn,real_fn)
self.workStatus=2
except:
print "writeBuffer err:",traceback.print_exc()
return
def isFinished(self):
if self.workStatus <= 1:
return False
return True
--
代碼下載： https://github.com/changshoumeng/python_local-network_file-transfer-service
--
--
]]>
#zhangtao 2016/06/14
#QQ 406878851
from PIL import *
import Image
import numpy as np
import os
########################################################################################
########################################################################################
class PixHelper(object):
def __init__(self ,(r ,g ,b)):
self. r =r
self. g =g
self. b =b
def grayVal(self):
return (self.r * 30 +self.g * 59 + self.b * 11) / 100
def validPix(self):
if self.isBackgroundPix():
if self.grayVal() != 255:
pass
# print self.grayVal()
return 255
return 0
def isBackgroundPix(self):
background = PixHelper((150, 150, 150))
background_grayVal = background.grayVal()
return self.grayVal() > background_grayVal
def isNeetPix(self):
return False == self.isBackgroundPix()
########################################################################################
########################################################################################
class ImageHelper(object):
model_file="svm.model"
def __init__(self, image):
self.image = image
self.width = image.size[0]
self.height = image.size[1]
self.subWidth=16
self.subHeight=self.height
#二值圖
def convertGrayImage(self):
for x in xrange(self.width):
for y in xrange(self.height):
pix = self.image.getpixel((x, y))
pixhelper = PixHelper(pix)
pix2 = pixhelper.validPix()
self.image.putpixel((x, y), (pix2, pix2, pix2))
def fixCropSubImages(self):
boxList = []
box0 = (7, 0, 20, self.height)
box1 = (20, 0, 34, self.height)
box2 = (34, 0, 46, self.height)
box3 = (46, 0, 59, self.height)
box4 = (59, 0, 72, self.height)
box5 = (72, 0, 88, self.height)
boxList.append(box0)
boxList.append(box1)
boxList.append(box2)
boxList.append(box3)
boxList.append(box4)
boxList.append(box5)
subList = []
for box in boxList:
sub = self.image.crop(box)
subList.append(sub)
return subList
def getRect(self,block):
minx = self.width
maxx = 0
miny = self.height
maxy = 0
for p in block:
# print p
minx = min(minx, p[0])
maxx = max(maxx, p[0])
miny = min(miny, p[1])
maxy = max(maxy, p[1])
return (minx, miny, maxx - minx + 1, maxy - miny + 1)
def getValidPoints(self):
points = list()
for x in xrange(self.width):
for y in xrange(self.height):
p = (x, y)
pix = self.image.getpixel(p)
pixhelper = PixHelper(pix)
if pixhelper.isNeetPix():
points.append(p)
pass
return points
def getDataListWithOffset(self):
points = self.getValidPoints()
(left, top, width, height) = self.getRect(points)
dataList = list()
# if 1 == 0:
# data = self.getData()
# dataList.append(data)
# return dataList
xBegin = 0 - left
xEnd = self.subWidth - (left + width)
yBegin = 0 - top
yEnd = self.subHeight - (top + height)
h_i = 0
w_i = 0
# print "######################"
for h_i in xrange(yBegin, yEnd, 1):
for w_i in xrange(xBegin, xEnd, 1):
data = [0] * (self.subWidth * self.subHeight)
for p in points:
(x, y) = p
(x2, y2) = (x + w_i, y + h_i)
if x2 >= self.subWidth or y2 >= self.subHeight:
continue
idx = x2 + y2 * self.subWidth
data[idx] = 1
dataList.append(data)
return dataList
def getData(self):
points = self.getValidPoints()
(left, top, width, height) = self.getRect(points)
data = [0] * (self.subWidth * self.subHeight)
for p in points:
(x, y) = p
idx = x + y * self.subWidth
data[idx] = 1
return data
----
# -*- coding: utf-8 -*-
#zhangtao 2016/06/14
#QQ 406878851
import os
import numpy as np
import mlpy
import sys
import time
from PIL import Image
from ImageHelper import *
from common import *
def checkSampleSucc(train_label, test_result):
if len(train_label) != len(test_result):
print "checkSampleSucc shape no match"
return
total = len(train_label)
succ = 0
for i in xrange(total):
item1 = test_result[i]
item2 = train_label[i]
if int(item1) == int(item2):
succ += 1
succRate = float(succ) / float(total)
print "checkSampleSucc : ", (succ, total), succRate
def getDataListFromFile(srcfilepath):
im = Image.open(srcfilepath).convert('RGB')
helper = ImageHelper(im)
return helper.getDataListWithOffset()
def loadSampleFromDir(label, srcdir, x, y):
print "load ..", srcdir
srcfiles = os.listdir(srcdir)
tm = int(time.time())
k = 0
for srcfilename in srcfiles:
srcfilepath = os.path.join(srcdir, srcfilename)
if srcfilename[0] == 'a':
pass
# continue
# print srcfilename
dataList = getDataListFromFile(srcfilepath)
idx = 0
for data in dataList:
x.append(data)
y.append(label)
# createImageFromData(srcdir,data,idx)
idx += 1
newfilename = "{0}\\{1}.png".format(srcdir, k + tm)
os.rename(srcfilepath,newfilename)
k += 1
def loadSampleFiles():
x = []
y = []
label_count = 0
for c in xrange(ord('a'), ord('z') + 1, 1):
if c == ord('i'):
pass
#continue
srcdir = r'sample\{0}'.format(chr(c))
#mkdir(srcdir)
loadSampleFromDir(c, srcdir, x, y)
label_count += 1
for i in xrange(1,10,1):
srcdir=r'sample\{0}'.format( i )
#mkdir(srcdir)
loadSampleFromDir(i,srcdir,x,y)
label_count += 1
print "label_count:",label_count
return (x, y)
def train():
(x, y) = loadSampleFiles()
train_data = np.array(x, dtype='float32')
train_label = np.array(y, dtype='int32')
svm = mlpy.LibSvm(svm_type='c_svc', kernel_type='poly', gamma=10, probability=True)
print "svm learn
."
print "len(x):",len(x)
svm.learn(train_data, train_label)
print "svm learn done!"
result = svm.pred(train_data)
print result
checkSampleSucc(y, result)
svm.save_model(ImageHelper.model_file)
# print result
def main():
begin = time.time()
train()
#test()
end = time.time()
t = end - begin
print "done:", t, t / (60)
# test2()
if __name__ == "__main__":
print __file__
main()
----#zhangtao 2016/06/14
#QQ 406878851
import os
import numpy as np
import mlpy
import sys
import time
from PIL import Image
from ImageHelper import *
from common import *
def checkSampleSucc(train_label, test_result):
if len(train_label) != len(test_result):
print "checkSampleSucc shape no match"
return
total = len(train_label)
succ = 0
for i in xrange(total):
item1 = test_result[i]
item2 = train_label[i]
if int(item1) == int(item2):
succ += 1
succRate = float(succ) / float(total)
print "checkSampleSucc : ", (succ, total), succRate
def getDataListFromFile(srcfilepath):
im = Image.open(srcfilepath).convert('RGB')
helper = ImageHelper(im)
return helper.getDataListWithOffset()
def loadSampleFromDir(label, srcdir, x, y):
print "load ..", srcdir
srcfiles = os.listdir(srcdir)
tm = int(time.time())
k = 0
for srcfilename in srcfiles:
srcfilepath = os.path.join(srcdir, srcfilename)
if srcfilename[0] == 'a':
pass
# continue
# print srcfilename
dataList = getDataListFromFile(srcfilepath)
idx = 0
for data in dataList:
x.append(data)
y.append(label)
# createImageFromData(srcdir,data,idx)
idx += 1
newfilename = "{0}\\{1}.png".format(srcdir, k + tm)
os.rename(srcfilepath,newfilename)
k += 1
def loadSampleFiles():
x = []
y = []
label_count = 0
for c in xrange(ord('a'), ord('z') + 1, 1):
if c == ord('i'):
pass
#continue
srcdir = r'sample\{0}'.format(chr(c))
#mkdir(srcdir)
loadSampleFromDir(c, srcdir, x, y)
label_count += 1
for i in xrange(1,10,1):
srcdir=r'sample\{0}'.format( i )
#mkdir(srcdir)
loadSampleFromDir(i,srcdir,x,y)
label_count += 1
print "label_count:",label_count
return (x, y)
def train():
(x, y) = loadSampleFiles()
train_data = np.array(x, dtype='float32')
train_label = np.array(y, dtype='int32')
svm = mlpy.LibSvm(svm_type='c_svc', kernel_type='poly', gamma=10, probability=True)
print "svm learn
."print "len(x):",len(x)
svm.learn(train_data, train_label)
print "svm learn done!"
result = svm.pred(train_data)
print result
checkSampleSucc(y, result)
svm.save_model(ImageHelper.model_file)
# print result
def main():
begin = time.time()
train()
#test()
end = time.time()
t = end - begin
print "done:", t, t / (60)
# test2()
if __name__ == "__main__":
print __file__
main()
# -*- coding: utf-8 -*-
#zhangtao 2016/06/14
#QQ 406878851
import os
import numpy as np
import mlpy
import sys
import time
from PIL import Image
from ImageHelper import *
from common import *
testDir="test"
def processOne(i):
dataList=[]
fn=r'img/{0}.jpg'.format(i)
fn2=r'img2/{0}.bmp'.format(i)
im = Image.open(fn)
helper = ImageHelper(im)
helper.convertGrayImage()
im.save(fn2)
subList=helper.fixCropSubImages()
count = len(subList)
for j in xrange(count):
fn3=r'img3/{0}_{1}.bmp'.format(i,j)
sub = subList[j]
sub.save(fn3)
subHelper=ImageHelper(sub)
data=subHelper.getData()
dataList.append(data)
return dataList
def savePredictResult(result_list):
global testDir
print "savePredictResult"
count=len(result_list)
if count > 0:
with open("data.txt",'w') as file:
for i in xrange(0,count,1):
item=result_list[i]
file.write(str(i)+":")
file.write(item)
file.write("\n")
#print i, item
lines1=[]
lines2=[]
with open("data.txt",'r') as f:
lines1=f.readlines()
with open(r"{0}/data.txt".format(testDir),'r') as f:
lines2=f.readlines()
count1=len(lines1)
count2=len(lines2)
if count1 != count2 :
print "no match :",count1,count2
return
succ=0
for i in xrange(count1):
line1=lines1[i].strip()
line2=lines2[i].strip()
if line1 != line2:
print "!=",line1,line2
else:
succ += 1
rate= float(succ)/float(count1)
print "result:",(succ,count1),rate
def main():
print "-----------test---------------------"
if False == os.path.exists(ImageHelper.model_file):
print "s.path.exists ,faild ", ImageHelper.model_file
return
svm = mlpy.LibSvm.load_model(ImageHelper.model_file)
#print 'nclasses:', svm.nclasses()
#print 'labels:', svm.labels()
test_list=[]
for i in xrange(100):
dataList=processOne(i)
labels=[]
for data in dataList:
input_data = np.array(data, dtype='float32')
result_label = int(svm.pred(input_data))
labels.append(result_label)
result_string=parseLabels(labels)
print "{0}:{1}".format(i,result_string)
test_list.append(result_string)
savePredictResult(test_list)
if __name__ == "__main__":
print __file__
main()
---#zhangtao 2016/06/14
#QQ 406878851
import os
import numpy as np
import mlpy
import sys
import time
from PIL import Image
from ImageHelper import *
from common import *
testDir="test"
def processOne(i):
dataList=[]
fn=r'img/{0}.jpg'.format(i)
fn2=r'img2/{0}.bmp'.format(i)
im = Image.open(fn)
helper = ImageHelper(im)
helper.convertGrayImage()
im.save(fn2)
subList=helper.fixCropSubImages()
count = len(subList)
for j in xrange(count):
fn3=r'img3/{0}_{1}.bmp'.format(i,j)
sub = subList[j]
sub.save(fn3)
subHelper=ImageHelper(sub)
data=subHelper.getData()
dataList.append(data)
return dataList
def savePredictResult(result_list):
global testDir
print "savePredictResult"
count=len(result_list)
if count > 0:
with open("data.txt",'w') as file:
for i in xrange(0,count,1):
item=result_list[i]
file.write(str(i)+":")
file.write(item)
file.write("\n")
#print i, item
lines1=[]
lines2=[]
with open("data.txt",'r') as f:
lines1=f.readlines()
with open(r"{0}/data.txt".format(testDir),'r') as f:
lines2=f.readlines()
count1=len(lines1)
count2=len(lines2)
if count1 != count2 :
print "no match :",count1,count2
return
succ=0
for i in xrange(count1):
line1=lines1[i].strip()
line2=lines2[i].strip()
if line1 != line2:
print "!=",line1,line2
else:
succ += 1
rate= float(succ)/float(count1)
print "result:",(succ,count1),rate
def main():
print "-----------test---------------------"
if False == os.path.exists(ImageHelper.model_file):
print "s.path.exists ,faild ", ImageHelper.model_file
return
svm = mlpy.LibSvm.load_model(ImageHelper.model_file)
#print 'nclasses:', svm.nclasses()
#print 'labels:', svm.labels()
test_list=[]
for i in xrange(100):
dataList=processOne(i)
labels=[]
for data in dataList:
input_data = np.array(data, dtype='float32')
result_label = int(svm.pred(input_data))
labels.append(result_label)
result_string=parseLabels(labels)
print "{0}:{1}".format(i,result_string)
test_list.append(result_string)
savePredictResult(test_list)
if __name__ == "__main__":
print __file__
main()
---
]]>
import sys
from Crypto.Cipher import AES
from binascii import b2a_hex, a2b_hex
import base64
def printMemLog2(mem):
byteData=bytearray(mem)
byteLen =len(byteData)
print "printMemLog begin
.byteLen:{0}".format(byteLen)i=0
while i <=byteLen-1:
a=byteData[i]
if i == byteLen-1:
print "%02x "%( a )
break;
i +=1
b=byteData[i]
print "%02x%02x "%( a,b),
i +=1
if i%64 == 0 :
print "*"*10
pass
#將text按照byteAlignLen字節對齊，如果不對齊，按照差數填充
def bytePad(text,byteAlignLen=16):
count=len(text)
mod_num=count%byteAlignLen
if mod_num==0:
return text
add_num=byteAlignLen-mod_num
print "bytePad:" ,add_num
return text+chr(add_num)*add_num
def byteUnpad(text,byteAlignLen=16):
count=len(text)
print "byteUnpad:",count
mod_num=count%byteAlignLen
assert mod_num==0
lastChar=text[-1]
lastLen=ord(lastChar)
lastChunk=text[-lastLen:]
if lastChunk==chr(lastLen)*lastLen:
#print "byteUnpad"
return text[:-lastLen]
return text
class prpcrypt():
def __init__(self, key):
self.key = key
self.mode = AES.MODE_ECB
self.iv = '\0' * 16
#加密函數，如果text不是16的倍數【加密文本text必須為16的倍數！】，那就補足為16的倍數
def encrypt(self, text):
cryptor = AES.new(self.key, self.mode, self.iv )
#這里密鑰key 長度必須為16（AES-128）、24（AES-192）、或32（AES-256）Bytes 長度.目前AES-128足夠用
#length = 16
text=bytePad(text,16)
#add = length - (count % length)
#text = text + ('\0' * add)
self.ciphertext = cryptor.encrypt(text)
#因為AES加密時候得到的字符串不一定是ascii字符集的，輸出到終端或者保存時候可能存在問題
#所以這里統一把加密后的字符串轉化為16進制字符串
return self.ciphertext
#return b2a_hex(self.ciphertext)
#解密后，去掉補足的空格用strip() 去掉
def decrypt(self, text):
cryptor = AES.new(self.key, self.mode, self.iv )
#plain_text = cryptor.decrypt(a2b_hex(text))
plain_text = cryptor.decrypt(text)
#printMemLog2(plain_text)
#print plain_text
return byteUnpad(plain_text)
]]>
#20160418 通過分析日志文件，對比redis的數據，更新redis的數據
import os
import redis
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
r=None
def byRedis(groupId,msgId):
global r
if r==None:
r = redis.Redis(host='192.168.1.227',port=6382,db=0)
print "-------------------------------------->redis"
if not r.ping() :
print "cannot ping redis"
return
key= 'groupmsgcounter_{0}'.format(groupId)
oldmsgId=r.get( key )
if oldmsgId == None:
oldmsgId=0
#print "redis error:",key,oldmsgId,msgId
oldmsgId=long(oldmsgId)
#print "redis:",key,oldmsgId,msgId
if oldmsgId < msgId:
print "less:",key,oldmsgId,msgId
return
newmsgId=max(msgId,oldmsgId )
if newmsgId == 0 :
return
print newmsgId
#r.set(key,newmsgId)
'''
cli-groupmsgid:124 svr-groupmsgid:124
groupid:72057594037938371
'''
def parse1(lineData):
flag1='cli-groupmsgid:'
flag2='svr-groupmsgid:'
flag3='lpContext:'
flag4='groupid:'
flag5='grouprealid:'
s=lineData
n1=s.find(flag1)
n2=s.find(flag2)
n3=s.find(flag3)
n4=s.find(flag4)
n5=s.find(flag5)
# print lineData
a=s[n1+len(flag1):n2]
b=s[n2++len(flag2):n3]
c=s[n4+len(flag4):n5]
a=long(a)
b=long(b)
c=long (c)
return (c,max(a,b))
#print n1,n2,n3
def paseFile(fileName):
result=dict()
with open(fileName) as f:
lines=f.readlines()
for line in lines:
lineData=line.strip()
(groupId,msgId) = parse1(lineData)
msgId2=msgId
if groupId in result:
msgId2=max(result[groupId],msgId)
result[groupId]=msgId2
# break
return result
def testTwoDict( result1,result2):
for (d,x) in result1.items():
msgId=x
if d in result2:
if msgId < result2[d] :
msgId=result2[d]
#print "find:",d,x,msgId
byRedis(d,msgId)
def main():
result1=paseFile("test1.txt")
result2=paseFile("test2.txt")
result3=dict()
testTwoDict( result1,result2)
testTwoDict( result2,result1)
main()
]]>
$content = file_get_contents('http://www.baidu.com/s?wd=zhongguoyihang&cl=3');
//shangzhengzhishu
//zhongguoyinhang
?>
<!DOCTYPE html>
<html>
<head lang="en">
<meta charset="UTF-8">
<meta content="text/html; charset=utf-8" http-equiv="Content-Type"/>
<meta content="no-cache,must-revalidate" http-equiv="Cache-Control">
<meta content="no-cache" http-equiv="pragma">
<meta content="0" http-equiv="expires">
<title>zthome</title>
<script type="text/javascript" src="resources/jquery/jquery-2.1.4.min.js"></script>
<script>
$(document).ready(function () {
$("#date").text((new Date()).toString());
setInterval("startRequest()",3000);
});
function startRequest()
{
window.location.reload();
}
</script>
<style type="text/css">
#gupiao {
width :200px;
}
</style>
</head>
<body>
<div id="date">
</div>
<div id="gupiao">
<?php echo $content ; ?>
</div>
</body>
</html>
]]>
from HttpRequestModule import *
import os
import json
import traceback
import codecs
from lxml import etree
import StringIO, gzip
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
def write_file(file_name,file_data,encoding):
if len(file_data) == 0 :
print "file_data is zero"
return
file_dir = r"D:\fs\test_data\qqzone"
file_path=os.path.join(file_dir,file_name)
print file_path
# fp=open(file_path,"w")
# fp.write(file_data)
# fp.flush()
# fp.close()
with codecs.open(file_path,"w",encoding) as f:
f.write(file_data)
def decodeJson(json_string):
decode_json=None
try:
decode_json=json.loads(json_string)
return decode_json
except (TypeError, ValueError) as err:
print( 'TypeError or ValueError:{0}'.format(err) )
except Exception,e:
print( traceback.format_exc() )
pass
return decode_json
def getUserBlogList():
blog_list=[]
diray_url='''
http://b1.qzone.qq.com/cgi-bin/blognew/get_abs?hostUin=859226880&blogType=0&cateName=&cateHex=&statYear=2015&reqInfo=7&pos=0&num=15&sortType=0&absType=0&source=0&rand=0.6346770680975169&ref=qzone&g_tk=1611717761&verbose=1
'''
data=doGet(diray_url)
data_len = len(data)
if data_len == 0 :
print "data len is 0"
return blog_list
data_json = data[10:data_len-2]
#write_file('bloglist.txt',data_json,'utf-8')
decode_json=decodeJson(data_json.decode("gbk"))
if decode_json == None :
print "decode_json is None"
return []
if decode_json['code'] != 0:
print "server response code is "+decode_json['code']
return []
data =decode_json['data']
if data['totalNum'] <=0 :
print "server response totalnum is "+data['totalNum']
return []
blog_list=data['list']
return blog_list
def getUserBlog(uin,blogid):
url='''
http://b1.qzone.qq.com/cgi-bin/blognew/blog_output_data?uin=%(uin)s&blogid=%(blogid)s&styledm=ctc.qzonestyle.gtimg.cn&imgdm=ctc.qzs.qq.com&bdm=b.qzone.qq.com&mode=2&numperpage=15×tamp=1437033537&dprefix=&inCharset=gb2312&outCharset=gb2312&ref=qzone
'''%{'uin':uin,'blogid':blogid}
my_headers={
"Accept-Encoding":"gzip,deflate,sdch",
"Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6" ,
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36" ,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" ,
"Referer": "http://ctc.qzs.qq.com/qzone/newblog/blogcanvas.html"
}
request = urllib2.Request(url,headers=my_headers)
try:
response = urllib2.urlopen(request)
except URLError,e:
if hasattr(e, 'code'):
print('The server couldn\'t fulfill the request. errorcode:{0}'.format(e.code ))
elif hasattr(e, 'reason'):
print('We failed to reach a server. reason:{0}'.format(e.reason ))
else:
page = response.read()
return page
return ""
def getText(elem):
rc = []
for node in elem.itertext():
rc.append(node.strip())
return ''.join(rc)
def gzdecode(data) :
compressedstream = StringIO.StringIO(data)
gziper = gzip.GzipFile(fileobj=compressedstream)
data2 = gziper.read() # 讀取解壓縮后數據
return data2
def test(blogid):
print blogid
blog_data=getUserBlog('859226880',blogid)
blog_data=gzdecode(blog_data)
#write_file( blogid+'.html',blog_data )
#return
try:
content=blog_data.decode('utf-8')
tree=etree.HTML(content)
node=tree.xpath("//div[@id='blogDetailDiv']")[0]
tgt_data=getText(node)
print "*"*30
print tgt_data
write_file( blogid+'.txt',tgt_data, 'gbk')
return
except Exception,ex :
print "111",Exception,":",ex
try:
content=blog_data.decode('gbk')
tree=etree.HTML(content)
node=tree.xpath("//div[@id='blogDetailDiv']")[0]
tgt_data=getText(node)
print "_"*30
print tgt_data
write_file( blogid+'.txt',tgt_data ,'utf-8')
except Exception,ex :
print "222",Exception,":",ex
def main():
print "main"
test("1288281044")
#return
blog_list=getUserBlogList()
for blog_item in blog_list:
blogId=blog_item['blogId']
print blogId
test( str(blogId) )
pass
main()
]]>
import socket
import struct
def IpStr2NetInt(IpStr):
return struct.unpack( "I",socket.inet_aton(IpStr) )[0]
def IpStr2HostInt(IpStr):
return socket.ntohl( IpStr2NetInt(IpStr) )
def NetInt2IpStr(NetInt):
#return HostInt2IpStr( socket.ntohl(NetInt) )
return socket.inet_ntoa( struct.pack('I',NetInt) )
def HostInt2IpStr(HostInt):
return NetInt2IpStr( socket.htonl(HostInt) )
def main():
ip="127.0.0.1"
print IpStr2NetInt(ip)
print IpStr2HostInt(ip)
print NetInt2IpStr( 16777343)
print HostInt2IpStr( 2130706433)
main()
]]>
]]>
:: 參考了
:: http://wenku.baidu.com/link?url=MjO8qNL3ttMb6gqjmprzXBxIWKvRwxSYjjDNGOSypuM-s5dKeuan1OOF7r3N6Fe6zXrqnrZmBuAB6xhn_Gxqk_
:: http://www.lixin.me/blog/2012/05/26/29536
:: zhangtao
:: 將此bat腳本，保存為xxx.bat，將NAME與LANG兩個變量設置為對應的值，然后存放在對應的目錄，即可用。
@set NAME=heilongjiang.ft.exp0
@set LANG=heilongjiang
@if not exist %NAME%.box ( goto LABLE_MAKEBOX ) else ( goto LABLE_TRIAN )
:LABLE_MAKEBOX
@echo -------begin,make box
-----------@pause
..\ocr.exe %NAME%.tif %NAME% -l eng batch.nochop makebox
@if not exist %NAME%.box goto END_FLAG
@echo -------end,make box ,ok-----------
@pause
exit
:LABLE_TRIAN
@echo -------begin,train
-----------@pause
..\ocr.exe %NAME%.tif %NAME% box.train
@pause
..\unicharset_extractor.exe %NAME%.box
@pause
echo ft 0 0 1 0 0 > %LANG%.font_properties
..\mftraining.exe -F %LANG%.font_properties -U unicharset %NAME%.tr
@pause
..\cntraining.exe %NAME%.tr
@pause
@rename normproto %LANG%.normproto
@rename unicharset %LANG%.unicharset
@rename inttemp %LANG%.inttemp
@rename pffmtable %LANG%.pffmtable
@rename shapetable %LANG%.shapetable
@pause
..\combine_tessdata.exe %LANG%.
@echo %NAME%.traineddata
@echo ------end,succ-------------
@pause
]]>
::請參考
:: http://www.kaiyuanba.cn/html/1/131/227/7891.htm
:: http://blog.wudilabs.org/entry/f25efc5f/
:: author: zhangtao/20140823 /QQ:406878851
::----------------------------------------------------------
@echo
@set NAME=heilongjiang
@if not exist %NAME%.box ( goto LABLE_MAKEBOX ) else ( goto LABLE_TRIAN )
:LABLE_MAKEBOX
@echo -------begin,make box
-----------@pause
tesseract %NAME%.tif %NAME% -l heilongjiang batch.nochop makebox
@if not exist %NAME%.box goto END_FLAG
@echo -------end,make box ,ok-----------
@pause
exit
:LABLE_TRIAN
@echo -------begin,train
-----------@pause
@echo -1-------------
tesseract %NAME%.tif %NAME% nobatch box.train
@if not exist %NAME%.tr goto END_FLAG
@echo -2-------------
unicharset_extractor %NAME%.box
mftraining -U unicharset -O %NAME%.unicharset %NAME%.tr
@if not exist %NAME%.unicharset goto END_FLAG
@echo -3-------------
cntraining %NAME%.tr
@rename normproto %NAME%.normproto
@rename Microfeat %NAME%.Microfeat
@rename inttemp %NAME%.inttemp
@rename pffmtable %NAME%.pffmtable
@echo -4-------------
combine_tessdata %NAME%.
@if not exist %NAME%.traineddata goto END_FLAG
@echo -----------------------------
@echo %NAME%.traineddata
@echo ------end,succ---------------
@pause
exit
:END_FLAG
@echo -------end,failed,-----------
@pause
exit
]]>
import os
from PIL import *
curdir="E:\\py\\WinPython-32bit-2.7.6.4\\study"
os.chdir(curdir)
def RGB2BlackWhite(filename):
im=Image.open(filename)
print "image info,",im.format,im.mode,im.size
(w,h)=im.size
R=0
G=0
B=0
for x in xrange(w):
for y in xrange(h):
pos=(x,y)
rgb=im.getpixel( pos )
(r,g,b)=rgb
R=R+r
G=G+g
B=B+b
rate1=R*1000/(R+G+B)
rate2=G*1000/(R+G+B)
rate3=B*1000/(R+G+B)
print "rate:",rate1,rate2,rate3
for x in xrange(w):
for y in xrange(h):
pos=(x,y)
rgb=im.getpixel( pos )
(r,g,b)=rgb
n= r*rate1/1000 + g*rate2/1000 + b*rate3/1000
#print "n:",n
if n>=90:
im.putpixel( pos,(255,255,255))
else:
im.putpixel( pos,(0,0,0))
im.save("blackwhite.bmp")
def saveAsBmp(fname):
pos1=fname.rfind('.')
fname1=fname[0:pos1]
fname1=fname1+'_2.bmp'
im = Image.open(fname)
new_im = Image.new("RGB", im.size)
new_im.paste(im)
new_im.save(fname1)
return fname1
if __name__=="__main__":
filename=saveAsBmp("4.bmp")
RGB2BlackWhite(filename)
這里用RGB明度值來做比較，效果更好，速度更快
#計算某一點的明度
def getLightness( (r,g,b)):
return (r*30+g*59+b*11)
]]>
import Imageimport sysimport oscurdir="E:\\py\\WinPython-32bit-2.7.6.4\\study"os.chdir(curdir)def processImage(infile): try: im = Image.open(infile) except IOError: print "Cant load", infile sys.exit(1) i = 0 mypalette = im.getpalette() try: while 1: im.putpalette(mypalette) new_im = Image.new("RGB", im.size) new_im.paste(im) new_im.save('foo'+str(i)+'.bmp') i += 1 im.seek(im.tell() + 1) except EOFError: pass # end of sequenceif __name__ == '__main__': processImage("10.gif") im=Image.open("foo0.bmp") print "img info:",im.format,im.size]]>
import osdef compare_linelist(filename1,filename2): print "compare_linelist (%s,%s)"%(filename1,filename2) f1=open(filename1,"r") f2=open(filename2,"r") line1=f1.readline().strip() line2=f2.readline().strip() nSucc=0 nCount=0 while True: if line1=='' or line2=='': break print "%s : %s"%(line1,line2) nCount=nCount+1 if line1==line2: nSucc=nSucc+1 line1=f1.readline().strip() line2=f2.readline().strip() rate=float(nSucc)/float(nCount)*100 f1.close() f2.close() return (nCount,nSucc,rate)if __name__=='__main__': work_directory="E:\\py\\WinPython-32bit-2.7.6.4\\study" os.chdir(work_directory) print "current directory:", work_directory filename1="1.txt" filename2="2.txt" result=compare_linelist(filename1,filename2) print("nCount:%d nSucc:%d rate:%d%%")%result]]>