consolidate Namespace Reference

Functions
list[str]	get_all_includes (conffwk.Configuration db, str file)
None	consolidate_db (str oksfile, str output_file, Optional[str] session_id=None)
Tuple[conffwk.Configuration, conffwk.Configuration]	create_copy_template (str oksfile, str output_file)
None	consolidate_full (conffwk.Configuration db, conffwk.Configuration new_db)
None	consolidate_session (conffwk.Configuration db, conffwk.Configuration new_db, str session_id)
list[conffwk.Configuration]	get_relationships (conffwk.Configuration db, conffwk.Configuration current_dal, list[conffwk.Configuration] dal_list)
None	copy_dals_to_cfg (conffwk.Configuration new_db, list[conffwk.Configuration] dal_list)
list[conffwk.Configuration]	copy_configuration (Path dest_dir, list[conffwk.Configuration] input_files)
None	consolidate_files (str oksfile, *input_files)

Variables
	log = getLogger('daqconf.consolidate')

Function Documentation

consolidate_db()

None consolidate_db	(	str	oksfile,
		str	output_file,
		Optional[str]	session_id = None )

Consolidates a session database files into a single file.

The consolidated file contains all the schema and data required to define the entire
session. If session_id is None, all sessions will be consolidated into the single
output file.

Args:
    oksfile: OKS file(s) to consolidate
    output_file: File to output consolidated database to 
    session_id: Name of session, defaults to None

Returns:
    None

Raises:
    None

Definition at line 32 of file consolidate.py.

) -> None:
    """
    Consolidates a session database files into a single file.
    
    The consolidated file contains all the schema and data required to define the entire
    session. If session_id is None, all sessions will be consolidated into the single
    output file.
 
    Args:
        oksfile: OKS file(s) to consolidate
        output_file: File to output consolidated database to 
        session_id: Name of session, defaults to None
 
    Returns:
        None
 
    Raises:
        None
    """    
    log.info(f"Consolidating database into output database '{output_file}'. Input database: '{oksfile}'.")
 
    sys.setrecursionlimit(10000)  # for example
    db, new_db = create_copy_template(oksfile, output_file)
 
    if session_id is None:
        log.debug("Consolidating all dals in %s into %s", oksfile, output_file)
        consolidate_full(db, new_db)
    else:
        log.debug("Consolidating all dals in session %s from %s into %s", session_id, oksfile, output_file)
 
        consolidate_session(db, new_db, session_id)
 
 

consolidate_files()

None consolidate_files	(	str	oksfile,
		*	input_files )

Definition at line 190 of file consolidate.py.

def consolidate_files(oksfile: str, *input_files) -> None:
    includes = []
    dbs = []
    str_in_files = '\n'.join(input_files)
    log.info(f"Consolidating {len(input_files)} databases into output database \'{oksfile}\'. Input databases: {str_in_files}")
    sys.setrecursionlimit(10000)  # for example
 
    for input_file in input_files:
        dbs.append(conffwk.Configuration("oksconflibs:" + input_file))
        includes += get_all_includes(dbs[len(dbs) - 1], None)
 
    includes = list(set(includes))
    includes = [i for i in includes if i not in input_files]
    log.debug(f"Included files: {includes}")
 
    new_db = conffwk.Configuration("oksconflibs")
    new_db.create_db(oksfile, includes)
 
    new_db.commit()
 
    for db in dbs:
        log.debug(f"Reading dal objects from old db {db}")
        dals = db.get_all_dals()
 
        log.debug(f"Copying objects to new db {new_db}")
        for dal in dals:
 
            try:
                new_db.get_dal(dals[dal].className(), dals[dal].id)
            except:
                new_db.add_dal(dals[dal])
            new_db.commit()
 
    log.debug(f"Saving database {new_db}")
    new_db.commit()

consolidate_full()

None consolidate_full	(	conffwk.Configuration	db,
		conffwk.Configuration	new_db )

Consolidates ALL dal objects in db into new_db

:param db: A conffwk.Configuration containing objects you want to copy over
:param new_db: A conffwk.Configuration you want to copy objects into

Definition at line 94 of file consolidate.py.

def consolidate_full(db: conffwk.Configuration, new_db: conffwk.Configuration)->None:
    """Consolidates ALL dal objects in db into new_db
 
    :param db: A conffwk.Configuration containing objects you want to copy over
    :param new_db: A conffwk.Configuration you want to copy objects into
    """    
    dal_list = list(db.get_all_dals().values())
    copy_dals_to_cfg(new_db, dal_list)
 

consolidate_session()

None consolidate_session	(	conffwk.Configuration	db,
		conffwk.Configuration	new_db,
		str	session_id )

Consolidates all objects related to the session with id 'session_id' into a single file

:param db: A conffwk.Configuration containing objects you want to copy over
:param new_db: A conffwk.Configuration you want to copy objects into
:param session_id: Name of session

Definition at line 103 of file consolidate.py.

def consolidate_session(db: conffwk.Configuration, new_db: conffwk.Configuration, session_id: str)->None:
    """
    Consolidates all objects related to the session with id 'session_id' into a single file
    
    :param db: A conffwk.Configuration containing objects you want to copy over
    :param new_db: A conffwk.Configuration you want to copy objects into
    :param session_id: Name of session
    """
 
    # Check session exists and load
    try:
        dal_session = db.get_dal('Session', session_id)
    except Exception as e:
        log.exception(e)
        raise e
    
    dal_list = get_relationships(db, dal_session, [])
    copy_dals_to_cfg(new_db, dal_list)
 

copy_configuration()

list[conffwk.Configuration] copy_configuration	(	Path	dest_dir,
		list[conffwk.Configuration]	input_files )

Definition at line 154 of file consolidate.py.

def copy_configuration(dest_dir : Path, input_files: list[conffwk.Configuration]) -> list[conffwk.Configuration]:
    if len(input_files) == 0:
        return []
 
    log.info(f"Copying configuration represented by databases: \'{input_files}\' to \'{dest_dir}\'")
    dest_dir = dest_dir.resolve() # Always include by absolute path when copying
    sys.setrecursionlimit(10000)  # for example
 
    output_dbs = []
 
    for input_file in input_files:
        db = conffwk.Configuration("oksconflibs:" + input_file)
        includes = db.get_includes(None)
        schemas = [i for i in includes if "schema.xml" in i]
        dbs = [i for i in includes if "data.xml" in i]
        newdbs = copy_configuration(dest_dir, dbs)
 
        output_file = dest_dir / os.path.basename(input_file)
 
        new_db = conffwk.Configuration("oksconflibs")
        new_db.create_db(str(output_file), schemas + newdbs)
        new_db.commit()
 
        dals = db.get_all_dals()
 
        for dal in dals:
            db.get_dal(dals[dal].className(), dals[dal].id)
            new_db.add_dal(dals[dal])
 
        new_db.commit()
        output_dbs.append(str(output_file))
    log.debug("DONE")
 
    return output_dbs
 
 

copy_dals_to_cfg()

None copy_dals_to_cfg	(	conffwk.Configuration	new_db,
		list[conffwk.Configuration]	dal_list )

Copy a list of dals into a configuration

Definition at line 142 of file consolidate.py.

def copy_dals_to_cfg(new_db: conffwk.Configuration, dal_list: list[conffwk.Configuration]) -> None:
    '''
    Copy a list of dals into a configuration
    '''
    log.debug("Copying %d objects to new db", len(dal_list))
    for dal in dal_list:
        new_db.add_dal(dal)
 
    log.debug("Saving database")
    new_db.commit()
    
    

create_copy_template()

Tuple[conffwk.Configuration, conffwk.Configuration] create_copy_template	(	str	oksfile,
		str	output_file )

Create a blank oks .data.xml template file with all the schema includes of oksfile.

Args:
    oksfile: OKS file to copy includes from
    output_file: OKS file to copy includes into

Returns:
    Tuple of old_db, copied_db

Definition at line 67 of file consolidate.py.

def create_copy_template(oksfile: str, output_file: str)->Tuple[conffwk.Configuration, conffwk.Configuration]:
    """
    Create a blank oks .data.xml template file with all the schema includes of oksfile.
 
    Args:
        oksfile: OKS file to copy includes from
        output_file: OKS file to copy includes into
 
    Returns:
        Tuple of old_db, copied_db
    """
    log.debug("Reading database")
    db = conffwk.Configuration("oksconflibs:" + oksfile)
 
    schemafiles = []
    includes = get_all_includes(db, None)
    schemafiles += [i for i in includes if "schema.xml" in i]
    log.debug(f"Included schemas: {schemafiles}")
 
    log.debug("Creating new database")
    new_db = conffwk.Configuration("oksconflibs")
    new_db.create_db(output_file, schemafiles)
    new_db.commit()
    
    return db, new_db
 
 

get_all_includes()

list[str] get_all_includes	(	conffwk.Configuration	db,
		str	file )

Recursively get all includes from an OKS database

Args:
    db (conffwk.Configuration): The OKS database to get includes from
    file (str): The file to get includes from

Returns:
    list(str): A list of all includes in the database

Raises:
    None

Definition at line 10 of file consolidate.py.

def get_all_includes(db: conffwk.Configuration, file: str) -> list[str]:
    """
    Recursively get all includes from an OKS database
 
    Args:
        db (conffwk.Configuration): The OKS database to get includes from
        file (str): The file to get includes from
 
    Returns:
        list(str): A list of all includes in the database
 
    Raises:
        None
    """
    includes = db.get_includes(file)
    for include in includes:
        if "data.xml" not in include:
            continue
        includes += get_all_includes(db, include)
 
    return list(set(includes))
 

get_relationships()

list[conffwk.Configuration] get_relationships	(	conffwk.Configuration	db,
		conffwk.Configuration	current_dal,
		list[conffwk.Configuration]	dal_list )

Recurssively get all objects related to current_dal

Definition at line 122 of file consolidate.py.

def get_relationships(db: conffwk.Configuration, current_dal: conffwk.Configuration, dal_list: list[conffwk.Configuration]) -> list[conffwk.Configuration]:
    '''
    Recurssively get all objects related to current_dal
    '''
    dal_list.append(current_dal)
    
    for rel in db.relations(current_dal.className(), all=True):
        rel_obj = getattr(current_dal, rel, None)
        if rel_obj is None:
            continue
        
        if not isinstance(rel_obj, list):
            rel_obj = [rel_obj]
        
        for rel_obj in rel_obj:
            dal_list = get_relationships(db, rel_obj, dal_list)
        
    return dal_list
 
 

Variable Documentation

log

consolidate.log = getLogger('daqconf.consolidate')

Definition at line 7 of file consolidate.py.