How we did network monitoring for 14,000 objects

<?xml version="1.0" encoding="ISO-8859-1"?><werks><WERKS>1234</WERKS><NAME1>4321-Пятерочка</NAME1><PLANT_IP>192.168.1.50</PLANT_IP><REGION>31</REGION><PSTLZ>308580</PSTLZ><CITY1>с.Мониторинговое</CITY1><CITY2>Москвоская обл.</CITY2><STREET>Ленина ул.</STREET><HOUSE_NUM1>1</HOUSE_NUM1><TEL_NUMBER>(999)777-77-77</TEL_NUMBER><BRANCH>CH</BRANCH><REGION>CH_MSK</REGION><REGION_NAME>Регион Центральный</REGION_NAME><FORMAT>CH_MSK_D</FORMAT><STATUS>Объект открыт</STATUS></werks>

#!/usr/bin/python3import sys, os, getopt, ipaddress
from datetime import datetime as dt
from zabbix.api import ZabbixAPI
import xml.etree.cElementTree as et
from report import report
import existhost
defping(ip):if os.system('ping -c 2 -W 1 %s > /dev/null'%ip) == 0:
        returnTrueelse:
        returnFalsedefmain(argv):global opath
    try:
        opts, args = getopt.getopt(argv,"hp:",["path="])
    except getopt.GetoptError:
        print('sync-sap-chg.py -p <path>')
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print('sync-sap-chg.py -p <path>')
            sys.exit()
        elif opt in ("-p", "--path"):
            opath = arg
defasynchronization(file, reportdata):
 zapi = ZabbixAPI(url='http://z.noc.x5.ru', user='user', password='pwd')
 #Константы, которые потом пригодятся
 left_kidney = '17855'
 right_kidney = '17856'
 f_type={'S':'Super','D':'DK','H':'Giper','A':'DK'}
 format={'D':'13','S':'14','H':'12','A':'13'}
 region={'CT':'21','UR':'19','SZ':'17',~omit~}
 try:
  #Начинаем разбор XML
  tree = et.ElementTree(file=file)
  root = tree.getroot()
  for werks in root.iter('werks'):
	#Создаем структуру словаря для объекта Zabbix
	interfaces=[{
	 'main':'1',
	 'type':'2',
	 'useip':'1',
	 'port':'161'
	}]
	shop={
	 'inventory':{},
	 'interfaces':interfaces,
	 'groups':[],
	 'templates':[{'templateid':'10194'}],
	 'inventory_mode':'1'
	}
	#Данные, которые будем искать в XML
	di = {
	 'WERKS':'',
	     ~omit~,
	 'STATUS':'Object Opened'
	}
	#Переводим все в словарьfor item in werks:
	  di[item.tag] = item.text
	  #Достаем из SAPID только цифрыif item.tag == 'WERKS':
	    n_proxy = ''.join(filter(lambda x: x.isdigit(), item.text))
	  #Находим IP роутера по адресу сервера. Подсеть везде /26try:
		ipaddress.ip_interface('%s/%s'%(di['PLANT_IP'].strip(),26))
		ip_chk = Trueexcept:
		ip_chk = False#Дальнейшая обработка, только если IP адрес верныйif (di['PLANT_IP'] != '1.1.1.1') and ip_chk:
		   #Записываем данные в словарьif di['FORMAT'][-1] in ['D','A','S','H']:
			  shop['inventory']['alias']=di['WERKS']
			  shop['inventory']['name']=di['NAME1']
			  shop['inventory']['poc_1_phone_a']=di['TEL_NUMBER']
			  shop['inventory']['location']=di['STREET']
		   #Особенности для каждого форматаif (di['FORMAT'][-1] in ['H']):
			  shop['host']=f_type[di['FORMAT'][-1]]+di['WERKS']
			  shop['interfaces'][0]['ip']=str(ipaddress.ip_interface('%s/%s'%(di['PLANT_IP'].strip(),24)).network[1])
			  #Добавляем ID шаблона мониторинга для объекта
			  shop['templates'][0]['templateid']='29529'elif (di['FORMAT'][-1] in ['S']):
			  ~omit~
		   #For D balance in proxiesif int(n_proxy[-1]) % 2 == 0:
			  shop['proxy_hostid'] = right_kidney
		   else:
			  shop['proxy_hostid'] = left_kidney
		   #В inventory тоже добавим IP
		   shop['inventory']['oob_ip']=shop['interfaces'][0]['ip']
		   #Назначение групп по словарям format и region
		   shop['groups']=[{'groupid':'9'}]
		   shop['groups'].append({'groupid':format[di['FORMAT'][-1]]})
		   shop['groups'].append({'groupid':region[di['FORMAT'][:2]]})
		   #Проверка статуса магазинаif di['STATUS'] == 'Объект открыт':
			  shop['status']='0'else:
			  shop['status']='1'#Записать текущую дату
		   T = dt.date(dt.now()).strftime("%d %B %Y")
		   shop['inventory']['date_hw_decomm'] = T
		   #Проверить есть ли такой хост в Zabbix
		   hostid = existhost.exist(shop['host'])
		   ip = shop['interfaces'][0]['ip']
		   #Если новый - создаем!if hostid == 0and shop['status'] == '0'and ping(ip):
			  zapi.host.create(shop)
			  reportdata['new'].append(di['WERKS'])
		   #Если уже есть - обновляем!elif hostid != 0and di['PLANT_IP'] != '1.1.1.1':
			  #Указываем какой HOSTID обновляем
			  shop['hostid']=hostid
			  #При обновлении нужно отрезать интерфейсы
			  shop.pop('interfaces')
			  zapi.host.update(shop)
			  reportdata['update'].append(di['WERKS'])
  #Переместить обработанный файл в архив
  os.system('mv %s /mnt/ftp/old_data/'%(file))
 except:
  reportdata['error'].append(di['WERKS'])
  report('Ошибка с файлом %s. Объект с ошибкой:%s'%(file, str(reportdata['error'])), 'SAP Sync Failed!')
  sys.exit()
defchecking(path):
	files = os.listdir(path)
	files.sort()
	reportdata ={'new':[], 'update':[], 'error':[]}
	for file in files:
		asynchronization(path+file, reportdata)
	if files != []:
		report('Файлы %s успешно синхронизированы! \n Добавлено %s объектов. \n Их sap:\n %s. \n Обновлено %s объектов. \n Их sap:\n %s'%(str(files), len(reportdata['new']), str(reportdata['new']), len(reportdata['update']), str(reportdata['update'])), 'SAP Sync Succeed!')
if __name__ == "__main__":
	main(sys.argv[1:])
	checking(opath)

#!/usr/bin/python3import sys, getopt
from zabbix.api import ZabbixAPI
defmain(argv):global aname
   try:
      opts, args = getopt.getopt(argv,"hn:",["name="])
   except getopt.GetoptError:
      print('existhost.py -n <name>')
      sys.exit(2)
   for opt, arg in opts:
      if opt == '-h':
         print('existhost.py -n <name>')
         sys.exit()
      elif opt in ("-n", "--name"):
         aname = arg
defexist(name):
   zapi = ZabbixAPI(url='http://z.noc.x5.ru/', user='user', password='pwd')
   hostget = zapi.host.get(search={'name':'%s'%name}, output='hostid')
   if hostget == []:
       return0else:
       return int(hostget[0]['hostid'])
if __name__ == "__main__":
        main(sys.argv[1:])
        print(exist(aname))

#!/usr/bin/python3# -*- coding: utf-8 -*-import smtplib, sys
from email.mime.text import MIMEText    
defreport(message, subject):
	me = 'zbx-scripts@x5.ru'
	you = 'mail@x5.ru'
	smtp_server = 'smtp.ru'
	msg = MIMEText(message)
	msg['Subject'] = subject
	msg['From'] = me
	msg['To'] = you
	s = smtplib.SMTP(smtp_server)
	s.sendmail(me, [you], msg.as_string())
	s.quit()
if __name__ == '__main__':
	report(sys.argv[2], sys.argv[1])

#!/usr/bin/python3# -*- coding: utf-8 -*-import sys, json, pysnmp, ipaddress, xlrd
from datetime import datetime as dt
from pysnmp.entity.rfc3413.oneliner import cmdgen
defsnmp(host, operation, *oid):
  generator = cmdgen.CommandGenerator()
  auth_data = cmdgen.UsmUserData('user', 'pwd', 'hash')
  transport = cmdgen.UdpTransportTarget((host, 161))
  getAtt = getattr(generator, '%sCmd'%operation)
  rst = (errorIndication, errorStatus, errorIndex, varBinds) = getAtt(auth_data, transport, *oid)
  ifnot errorIndication isNoneor errorStatus isTrue:
     return"Error: %s %s %s %s" % rst
  else:
     if operation=='get':
        return varBinds
     elif operation=='next':
        result=[]
        for var in varBinds:
           result.append(var)
        return result
defxlsdata(file, sap):
  rb = xlrd.open_workbook(file)
  sheet = rb.sheet_by_index(0)
  base = {}
  for i in range(0, sheet.nrows-1):
     sapnum = str(round(sheet.cell(i,4).value)) if isinstance(sheet.cell(i,4).value,float) else sheet.cell(i,4).value
     name = str(round(sheet.cell(i,8).value)) if isinstance(sheet.cell(i,8).value,float) else sheet.cell(i,8).value
     if sap.upper() == sapnum.upper():
        base = {
          'type'		: (sheet.cell(i,2).value),
          'serialno_a'	: (sheet.cell(i,13).value),
          'serialno_b'	: (sheet.cell(i,20).value),
          'tag'		: ('2') if sheet.cell(i, 20).value != ''else ('1'),
          'macaddress_a'	: (sheet.cell(i, 15).value),
          'macaddress_b'	: (sheet.cell(i, 22).value)
        }
        base['date_hw_purchase'] = dt.date(dt.now()).strftime("%d %B %Y")
  return (base)
definventory(host, sap):
  BGPASBASE={}
  for line in open('/path/a.prokhorov/integration/BGP-AS-BASE.cfg'):
     if':'in line:
        line = line.split(':')
        BGPASBASE[line[0]]='%s(%s)'%(line[1].rstrip(), line[0])
  ### Get Data from Operator
  shop = xlsdata('/mnt/oprf/providers_base.xlsx', sap)
  shop['date_hw_expiry'] = 'Failed'### Get SNMP data
  shop['host_router'] = 'None'
  shop['host_netmask'] = 'None'
  shop['host_networks'] = 'None'try:
     ### Get Networks from router
     networks = ''for ip,mask in snmp(host, 'next', 'iso.3.6.1.2.1.4.20.1.1', 'iso.3.6.1.2.1.4.20.1.3'):
        networks = networks+str(ipaddress.ip_interface(u'%s/%s'%(ip[1].prettyPrint(), mask[1].prettyPrint())))+'\n'
     shop['host_networks']=networks
     ### Get BGP information
     bgppeers, ispnames = '',''for peer,asbgp in snmp(host, 'next', 'iso.3.6.1.2.1.15.3.1.7', 'iso.3.6.1.2.1.15.3.1.9'):
        asbgp = asbgp[1].prettyPrint()
        bgppeers = bgppeers+peer[1].prettyPrint()+'\n'
        ispnames = ispnames+(BGPASBASE.get(asbgp) if BGPASBASE.get(asbgp)!=Noneelse asbgp)+'\n'
     shop['host_router'] = bgppeers.strip()[:38]
     shop['host_netmask'] = ispnames.strip()[:38]
     ### Get Vendor name and Model type
     hardware = snmp(host, 'get', 'iso.3.6.1.2.1.47.1.1.1.1.13.1', 'iso.3.6.1.2.1.47.1.1.1.1.10.1', 'iso.3.6.1.2.1.47.1.1.1.1.12.1', 'iso.3.6.1.2.1.1.1.0', 'iso.3.6.1.2.1.47.1.1.1.1.7.1')
     if str(hardware[0][1]) == '0235A325':
        shop['model'] = hardware[4][1].prettyPrint()
     else:
        shop['model'] = hardware[0][1].prettyPrint()
     shop['os_short'] = hardware[1][1].prettyPrint()
     shop['vendor'] = hardware[2][1].prettyPrint()
     version = hardware[3][1].prettyPrint()
     os = version.split('\n')[0]
     shop['os_full'] = version[:250]
     shop['os'] = ''.join(os.split(',')[:2])[:60]
     ### Make indicators
     shop['date_hw_expiry'] = 'Success'
     shop['date_hw_install'] = dt.date(dt.now()).strftime("%d %B %Y")
  except:
     shop.pop('host_router')
     shop.pop('host_netmask')
     shop.pop('host_networks')
  return shop
  #return json.dumps(dict([('inventory',shop)]), sort_keys=True, indent=4)if __name__ == "__main__":
  print(inventory(sys.argv[1], sys.argv[2]))

#!/usr/bin/python3# -*- coding: utf-8 -*-from multiprocessing import Pool
import time
from zabbix.api import ZabbixAPI
from inventory import inventory
from report import report
defupdating(shop):try:
    shop['inventory'] = inventory(shop['interfaces'][0]['ip'], shop['host'][-4:])
    shop.pop('interfaces')
    shop['inventory_mode'] = '1'
    shop.pop('host')
    print (shop['hostid'])
    return zapi.host.update(shop)
  except:
    print(">>>",shop['hostid'])
    with open ('/home/local/integration/error.txt', 'a') as err:
       err.write(shop['hostid'])
       err.write("\n")
if __name__ =='__main__':
  t = time.time()
  zapi = ZabbixAPI(url='http://z.noc.x5.ru/', user='user', password='pwd')	
  shopbase = zapi.host.get(output=['host', 'hostid'], groupids= ['12', '13', '14'], monitored="1", selectInterfaces=['ip'])
  pool = Pool(processes=10)
  p=[0for x in range(0,len(shopbase))]
  for i in range(0, len(shopbase), 10):
     print ("Index:", i,"\n",shopbase[i],"\n")
     pool.map(updating,shopbase[i:i+10])
  pool.close()
  pool.join()
  print(time.time()-t)
  report('Инвентарные данные обновлены в Zabbix.', 'Inventory updating succeed')