manage/uuspider/create_table/testInsertcomponent.py

#coding=utf-8


'''
Created on 2016年6月7日

@author: uas
'''


import os 
import time
os.environ['NLS_LANG']='SIMPLIFIED CHINESE_CHINA.UTF8'
from pymongo.mongo_client import MongoClient

import cx_Oracle
from util_common import Constant

class OracleManager(object):
    def __init__(self):
        self.conn=cx_Oracle.connect(Constant.ORACLE_URL)

    
    
    def _insert_component_temp(self,component_list):
        
        cursor=self.conn.cursor()
#         sql="create table product$component_temp1 (cmp_id  int primary key,cmp_attach  varchar(255) ,cmp_img varchar(255),cmp_brid  int ,cmp_code   varchar(255),cmp_kiid  int,cmp_company  varchar(255) ,cmp_company_url varchar(255) ,cmp_description varchar(4000),cmp_packaging varchar(255))"

        sql="drop table PRODUCT$COMPONENT_TEMP1"
        cursor.execute(sql)
        self.conn.commit()




#         print(self.conn.version)
#         cursor.prepare("INSERT INTO product$component_temp1 (cmp_id,cmp_attach,cmp_img,cmp_brid,cmp_code,cmp_kiid,cmp_company,cmp_company_url,cmp_description,cmp_packaging) values(:1,:2,:3,:4,:5,:6,:7,:8,:9,:10)")      
#         param_oracle=list()
#         error_component_list=list()
#         for index,component in enumerate(component_list):
#             try:
#                 param_oracle.append((component['cmp_id'],component['cmp_attach'],component['cmp_img'],component['cmp_brid'],component['cmp_code'],component['cmp_kiid'],component['cmp_company'],component['cmp_company_url'],component['description'],component['cmp_packaging']))
#             except Exception as e:
#                 error_component_list.append(component)
# 
#                  
#             if index %20000 ==0:
#                  
#                 cursor.executemany(None,param_oracle)
#                  
#                 self.conn.commit()
#                 param_oracle.clear()
#             print(index)
#             if index>10000:
#                 break
#         print( len(error_component_list))
#         print(2)
#         print(param_oracle)
#         cursor.executemany(None,param_oracle)
#         print(1)
#         self.conn.commit()
#         print(3)
#         cursor.close()
#          
    def craw(self,component_list,propertyvalue_list):
        self._insert_component_temp(component_list)
#         self._insert_property_value(propertyvalue_list)
    
    
    
#     
#     def _insert_property_value(self,propertyvalue_list):
#         cursor=self.conn.cursor()
#         cursor.prepare('insert into product$propertyvalue_temp (pv_id,pc_componentid,pv_detno,pv_propertyid,pv_value) values(:1,:2,:3,:4,:5)')
#         param_oracle=list()
#         for index,pv in enumerate(propertyvalue_list):
#             try:
#                 if len(pv["value"])>80:
#                     continue
#                 param_oracle.append((index+1,pv['cmp_id'],pv['detno']-1,pv['propertyid'],pv['value']))
#             except:
#                 param_oracle.append((index+1,pv['cmp_id'],pv['detno']-1,pv['propertyid'],None))
#             if index%50000==0:
#                 cursor.executemany(None,param_oracle)
#                 param_oracle.clear()
#              
#             cursor.executemany(param_oracle)
#             self.conn.commit()
#             cursor.close()
            
if __name__=='__main__':
    Oracle=OracleManager()
    cli=MongoClient(Constant.MONGODB_URL)
    db=cli.spider
    component_list=db.component_temp_0607.find().limit(100)
    component_list_temp=list()
    for row in component_list:
        component_list_temp.append(row)
        print(row)
#     propertyvalue_list=db.propertyvalue_0607.find()
    Oracle.craw(component_list_temp,None)