consolidate Namespace Reference

Functions
	get_all_includes (db, file)
None	consolidate_db (str oksfile, str output_file, Optional[str] session_id=None)
Tuple[conffwk.Configuration, conffwk.Configuration]	create_copy_template (str oksfile, str output_file)
None	consolidate_full (conffwk.Configuration db, conffwk.Configuration new_db)
None	consolidate_session (conffwk.Configuration db, conffwk.Configuration new_db, str session_id)
	get_relationships (conffwk.Configuration db, current_dal, dal_list)
None	copy_dals_to_cfg (conffwk.Configuration new_db, dal_list)
	copy_configuration (Path dest_dir, list input_files)
	consolidate_files (oksfile, *input_files)

Variables
	log = getLogger('daqconf.consolidate')

Function Documentation

consolidate_db()

None consolidate.consolidate_db	(	str	oksfile,
		str	output_file,
		Optional[str]	session_id = None )

Consolidates a single session

:param oksfile: OKS file(s) to consolidate
:param output_file: File to output consolidated database to 
:param session_id: Name of session, defaults to None

Definition at line 19 of file consolidate.py.

def consolidate_db(oksfile: str, output_file: str, session_id: Optional[str] = None)->None:
    """Consolidates a single session
 
    :param oksfile: OKS file(s) to consolidate
    :param output_file: File to output consolidated database to 
    :param session_id: Name of session, defaults to None
    """    
    log.info(f"Consolidating database into output database '{output_file}'. Input database: '{oksfile}'.")
 
    sys.setrecursionlimit(10000)  # for example
    db, new_db = create_copy_template(oksfile, output_file)
 
    if session_id is None:
        log.debug("Consolidating all dals in %s into %s", oksfile, output_file)
        consolidate_full(db, new_db)
    else:
        log.debug("Consolidating all dals in session %s from %s into %s", session_id, oksfile, output_file)
 
        consolidate_session(db, new_db, session_id)
 
 

consolidate_files()

consolidate.consolidate_files	(		oksfile,
		*	input_files )

Definition at line 160 of file consolidate.py.

def consolidate_files(oksfile, *input_files):
    includes = []
    dbs = []
    str_in_files = '\n'.join(input_files)
    log.info(f"Consolidating {len(input_files)} databases into output database \'{oksfile}\'. Input databases: {str_in_files}")
    sys.setrecursionlimit(10000)  # for example
 
    for input_file in input_files:
        dbs.append(conffwk.Configuration("oksconflibs:" + input_file))
        includes += get_all_includes(dbs[len(dbs) - 1], None)
 
    includes = list(set(includes))
    includes = [i for i in includes if i not in input_files]
    log.debug(f"Included files: {includes}")
 
    new_db = conffwk.Configuration("oksconflibs")
    new_db.create_db(oksfile, includes)
 
    new_db.commit()
 
    for db in dbs:
        log.debug(f"Reading dal objects from old db {db}")
        dals = db.get_all_dals()
 
        log.debug(f"Copying objects to new db {new_db}")
        for dal in dals:
 
            try:
                new_db.get_dal(dals[dal].className(), dals[dal].id)
            except:
                new_db.add_dal(dals[dal])
            new_db.commit()
 
    log.debug(f"Saving database {new_db}")
    new_db.commit()

consolidate_full()

None consolidate.consolidate_full	(	conffwk.Configuration	db,
		conffwk.Configuration	new_db )

Consolidates ALL dal objects in db into new_db

:param db: A conffwk.Configuration containing objects you want to copy over
:param new_db: A conffwk.Configuration you want to copy objects into

Definition at line 64 of file consolidate.py.

def consolidate_full(db: conffwk.Configuration, new_db: conffwk.Configuration)->None:
    """Consolidates ALL dal objects in db into new_db
 
    :param db: A conffwk.Configuration containing objects you want to copy over
    :param new_db: A conffwk.Configuration you want to copy objects into
    """    
    dal_list = list(db.get_all_dals().values())
    copy_dals_to_cfg(new_db, dal_list)
 

consolidate_session()

None consolidate.consolidate_session	(	conffwk.Configuration	db,
		conffwk.Configuration	new_db,
		str	session_id )

Consolidates all objects related to the session with id 'session_id' into a single file

:param db: A conffwk.Configuration containing objects you want to copy over
:param new_db: A conffwk.Configuration you want to copy objects into
:param session_id: Name of session

Definition at line 73 of file consolidate.py.

def consolidate_session(db: conffwk.Configuration, new_db: conffwk.Configuration, session_id: str)->None:
    """
    Consolidates all objects related to the session with id 'session_id' into a single file
    
    :param db: A conffwk.Configuration containing objects you want to copy over
    :param new_db: A conffwk.Configuration you want to copy objects into
    :param session_id: Name of session
    """
 
    # Check session exists and load
    try:
        dal_session = db.get_dal('Session', session_id)
    except Exception as e:
        log.exception(e)
        raise e
    
    dal_list = get_relationships(db, dal_session, [])
    copy_dals_to_cfg(new_db, dal_list)
 

copy_configuration()

consolidate.copy_configuration	(	Path	dest_dir,
		list	input_files )

Definition at line 124 of file consolidate.py.

def copy_configuration(dest_dir : Path, input_files: list):
    if len(input_files) == 0:
        return []
 
    log.info(f"Copying configuration represented by databases: \'{input_files}\' to \'{dest_dir}\'")
    dest_dir = dest_dir.resolve() # Always include by absolute path when copying
    sys.setrecursionlimit(10000)  # for example
 
    output_dbs = []
 
    for input_file in input_files:
        db = conffwk.Configuration("oksconflibs:" + input_file)
        includes = db.get_includes(None)
        schemas = [i for i in includes if "schema.xml" in i]
        dbs = [i for i in includes if "data.xml" in i]
        newdbs = copy_configuration(dest_dir, dbs)
 
        output_file = dest_dir / os.path.basename(input_file)
 
        new_db = conffwk.Configuration("oksconflibs")
        new_db.create_db(str(output_file), schemas + newdbs)
        new_db.commit()
 
        dals = db.get_all_dals()
 
        for dal in dals:
            db.get_dal(dals[dal].className(), dals[dal].id)
            new_db.add_dal(dals[dal])
 
        new_db.commit()
        output_dbs.append(str(output_file))
    log.debug("DONE")
 
    return output_dbs
 
 

copy_dals_to_cfg()

None consolidate.copy_dals_to_cfg	(	conffwk.Configuration	new_db,
			dal_list )

Copy a list of dals into a configuration

Definition at line 112 of file consolidate.py.

def copy_dals_to_cfg(new_db: conffwk.Configuration, dal_list)->None:
    '''
    Copy a list of dals into a configuration
    '''
    log.debug("Copying %d objects to new db", len(dal_list))
    for dal in dal_list:
        new_db.add_dal(dal)
 
    log.debug("Saving database")
    new_db.commit()
    
    

create_copy_template()

Tuple[conffwk.Configuration, conffwk.Configuration] consolidate.create_copy_template	(	str	oksfile,
		str	output_file )

Creates a blank oks .data.xml file stored in output_file with all the schema includes of oksfile
:param oksfile: OKS file to copy includes from
:param output_file: OKS file to copy includes into

:returns: Tuple of old_db, copied_db

Definition at line 40 of file consolidate.py.

def create_copy_template(oksfile: str, output_file: str)->Tuple[conffwk.Configuration, conffwk.Configuration]:
    '''
    Creates a blank oks .data.xml file stored in output_file with all the schema includes of oksfile
    :param oksfile: OKS file to copy includes from
    :param output_file: OKS file to copy includes into
 
    :returns: Tuple of old_db, copied_db
    '''
    log.debug("Reading database")
    db = conffwk.Configuration("oksconflibs:" + oksfile)
 
    schemafiles = []
    includes = get_all_includes(db, None)
    schemafiles += [i for i in includes if "schema.xml" in i]
    log.debug(f"Included schemas: {schemafiles}")
 
    log.debug("Creating new database")
    new_db = conffwk.Configuration("oksconflibs")
    new_db.create_db(output_file, schemafiles)
    new_db.commit()
    
    return db, new_db
 
 

get_all_includes()

consolidate.get_all_includes	(		db,
			file )

Definition at line 10 of file consolidate.py.

def get_all_includes(db, file):
    includes = db.get_includes(file)
    for include in includes:
        if "data.xml" not in include:
            continue
        includes += get_all_includes(db, include)
 
    return list(set(includes))
 

get_relationships()

consolidate.get_relationships	(	conffwk.Configuration	db,
			current_dal,
			dal_list )

Recurssively get all objects related to current_dal

Definition at line 92 of file consolidate.py.

def get_relationships(db: conffwk.Configuration, current_dal, dal_list):
    '''
    Recurssively get all objects related to current_dal
    '''
    dal_list.append(current_dal)
    
    for rel in db.relations(current_dal.className(), all=True):
        rel_obj = getattr(current_dal, rel, None)
        if rel_obj is None:
            continue
        
        if not isinstance(rel_obj, list):
            rel_obj = [rel_obj]
        
        for rel_obj in rel_obj:
            dal_list = get_relationships(db, rel_obj, dal_list)
        
    return dal_list
 
 

Variable Documentation

log

consolidate.log = getLogger('daqconf.consolidate')

Definition at line 7 of file consolidate.py.