Oracle Data Integrator (ODI) - useful queries, management, admin

ODI - Repo - Warnings (SLT).sql

-- Errors en el sentit de Warnings per files desviades a taules d'error E$.
-- Connexió: BISLT_PRO_WORKREP
select
--sc_par_par_folder.SCEN_FOLDER_NAME
  sc_parent_folder.SCEN_FOLDER_NAME
, SNP_SCEN_REPORT.CONTEXT_CODE
, SNP_SCEN_REPORT.SESS_BEG
, ROUND(SNP_SCEN_REPORT.SESS_DUR/60) as DUR_MINS
, SNP_SCEN.SCEN_NAME
, SNP_SCEN.SCEN_VERSION
, SNP_SCEN_REPORT.NB_ERR
, SNP_SCEN_REPORT.NB_INS
, SNP_SCEN_REPORT.NB_UPD
, SNP_SCEN_REPORT.NB_ROW
from BISLT_PRO_WORKREP.SNP_SCEN_REPORT
    inner join BISLT_PRO_WORKREP.SNP_SCEN on SNP_SCEN.SCEN_NO=SNP_SCEN_REPORT.SCEN_NO
      inner join BISLT_PRO_WORKREP.SNP_SCEN_FOLDER on SNP_SCEN_FOLDER.I_SCEN_FOLDER=SNP_SCEN.I_SCEN_FOLDER
        left join BISLT_PRO_WORKREP.SNP_SCEN_FOLDER sc_parent_folder on sc_parent_folder.I_SCEN_FOLDER=SNP_SCEN_FOLDER.PAR_I_SCEN_FOLDER
          left join BISLT_PRO_WORKREP.SNP_SCEN_FOLDER sc_par_par_folder on sc_par_par_folder.I_SCEN_FOLDER=sc_parent_folder.PAR_I_SCEN_FOLDER
where 1=1
and SNP_SCEN_REPORT.NB_ERR>0
order by SESS_BEG desc
;

ODI query repository executions.sql

https://community.oracle.com/thread/4279794

select  distinct owner from all_tables;
-- Adaptadas de: https://community.oracle.com/thread/4279794


select * FROM  ETQPRO_BISLT_REPO.SNP_SESSION SS where 1=1 AND SS.SESS_BEG > TO_DATE('20200715','yyyymmdd')
 and SS.SESS_STATUS IN ('E', 'M')
 ORDER BY SS.SCEN_NAME ASC, SS.SESS_BEG ASC;


-- RUNTIME

SELECT SS.SESS_NO
     , SS.SCEN_NAME
     , SS.SCEN_VERSION
     , SS.SESS_NAME
     , SS.PARENT_SESS_NO
     , SS.SESS_BEG
     , SS.SESS_END
     , SS.SESS_STATUS
     , DECODE(SS.SESS_STATUS, 'D', 'Done', 'E', 'Error', 'M', 'Warning', 'Q', 'Queued', 'R', 'Running', 'W', 'Waiting', SS.SESS_STATUS) AS SESS_STATUS_DESC
     , SSL.NNO
     , SSTL.NB_RUN
     , SST.TASK_TYPE
     , DECODE(SST.TASK_TYPE, 'C', 'Loading', 'J', 'Mapping', 'S', 'Procedure', 'V', 'Variable', SST.TASK_TYPE) AS TASK_TYPE_DESC
     , SST.EXE_CHANNEL
     , DECODE(SST.EXE_CHANNEL, 'B', 'Oracle Data Integrator Scripting', 'C', 'Oracle Data Integrator Connector', 'J', 'JDBC', 'O', 'Operating System', 'Q', 'Queue', 'S', 'Oracle Data Integrator Command', 'T', 'Topic', 'U', 'XML Topic', SST.EXE_CHANNEL) AS EXE_CHANNEL_DESC
     , SSTL.SCEN_TASK_NO
     , SST.PAR_SCEN_TASK_NO
     , SST.TASK_NAME1
     , SST.TASK_NAME2
     , SST.TASK_NAME3
     , SSTL.TASK_DUR
     , SSTL.NB_ROW
     , SSTL.NB_INS
     , SSTL.NB_UPD
     , SSTL.NB_DEL
     , SSTL.NB_ERR
     , SSS.LSCHEMA_NAME || '.' || SSS.RES_NAME AS TARGET_TABLE
     , CASE WHEN SST.COL_TECH_INT_NAME IS NOT NULL AND SST.COL_LSCHEMA_NAME IS NOT NULL THEN SST.COL_TECH_INT_NAME || '.' || SST.COL_LSCHEMA_NAME
            ELSE NULL
       END AS TARGET_SCHEMA
     , SSTL.DEF_TXT AS TARGET_COMMAND
     , CASE WHEN SST.DEF_TECH_INT_NAME IS NOT NULL AND SST.DEF_LSCHEMA_NAME IS NOT NULL THEN SST.DEF_TECH_INT_NAME || '.' || SST.DEF_LSCHEMA_NAME
            ELSE NULL
       END AS SOURCE_SCHEMA
     , SSTL.COL_TXT AS SOURCE_COMMAND
  FROM ETQPRO_BISLT_REPO.SNP_SESSION SS
    INNER JOIN ETQPRO_BISLT_REPO.SNP_STEP_LOG SSL
        ON SS.SESS_NO = SSL.SESS_NO
    INNER JOIN ETQPRO_BISLT_REPO.SNP_SESS_TASK_LOG SSTL
        ON SS.SESS_NO = SSTL.SESS_NO
    INNER JOIN ETQPRO_BISLT_REPO.SNP_SB_TASK SST
        ON SSTL.SB_NO = SST.SB_NO
        AND SSTL.SCEN_TASK_NO = SST.SCEN_TASK_NO
        AND SSL.NNO = SSTL.NNO
        AND SSTL.NNO = SST.NNO
        AND SSL.NB_RUN = SSTL.NB_RUN
    LEFT JOIN ETQPRO_BISLT_REPO.SNP_SB_STEP SSS
        ON SST.SB_NO = SSS.SB_NO
        AND SST.NNO = SSS.NNO
WHERE 1 = 1
   AND SS.SESS_BEG > TO_DATE('20200715','yyyymmdd')
   -- AND SS.SESS_NO = 123542
   AND SST.EXE_CHANNEL IN ('B', 'C', 'J', 'S')
   AND (UPPER(SSTL.DEF_TXT) LIKE '%SELECT%' OR UPPER(SSTL.DEF_TXT) LIKE '%MERGE%' OR UPPER(SSTL.DEF_TXT) LIKE '%INSERT%')
   AND SST.TASK_TYPE NOT IN ('V') --VARIABLE
   
   -- AND SS.SESS_STATUS IN ('D', 'R', 'M', 'E')   
    and (SS.SESS_STATUS = 'E'
        OR (SS.SESS_STATUS = 'M' AND SSTL.NB_ERR > 0) -- IN ('D', 'R', 'M', 'E')   
    )
   
--ORDER BY SS.SESS_NO DESC, SSL.NNO, SSL.NB_RUN, SSTL.SCEN_TASK_NO;
ORDER BY SS.SCEN_NAME ASC, SS.SESS_BEG ASC, SSL.NNO, SSL.NB_RUN, SSTL.SCEN_TASK_NO;
 
 


 

-- MAPPING

WITH

MOD_FOLDERS__ AS (

SELECT F.I_MOD_FOLDER, F.MOD_FOLDER_NAME, LEVEL AS LEVEL_

     , SYS_CONNECT_BY_PATH(MOD_FOLDER_NAME, ' -> ') AS MOD_FOLDER_PATH

  FROM DEV_ODI_REPO.SNP_MOD_FOLDER F

  CONNECT BY PRIOR F.I_MOD_FOLDER = F.PAR_I_MOD_FOLDER

)

, MOD_FOLDERS_MAX__ AS (

SELECT I_MOD_FOLDER, MAX(LEVEL_) AS LEVEL_

  FROM MOD_FOLDERS__ F

GROUP BY I_MOD_FOLDER

) 

, MOD_FOLDERS_ AS (

SELECT SUBSTR(MF.MOD_FOLDER_PATH, 5) AS MOD_FOLDER_PATH, MF.I_MOD_FOLDER, MF.MOD_FOLDER_NAME, MF.LEVEL_

  FROM MOD_FOLDERS__ MF

    INNER JOIN MOD_FOLDERS_MAX__ M

        ON MF.I_MOD_FOLDER = M.I_MOD_FOLDER

        AND MF.LEVEL_ = M.LEVEL_

)

, FOLDERS__ AS (

SELECT F.I_FOLDER, F.FOLDER_NAME, F.I_PROJECT, LEVEL AS LEVEL_

     , SYS_CONNECT_BY_PATH(FOLDER_NAME, ' -> ') AS FOLDER_PATH

  FROM DEV_ODI_REPO.SNP_FOLDER F

  WHERE F.I_PROJECT = 1

  CONNECT BY PRIOR F.I_FOLDER = F.PAR_I_FOLDER

)

, FOLDERS_MAX__ AS (

SELECT I_FOLDER, MAX(LEVEL_) AS LEVEL_

  FROM FOLDERS__ F

GROUP BY I_FOLDER

) 

, FOLDERS_ AS (

SELECT SUBSTR(F.FOLDER_PATH, 5) AS FOLDER_PATH, F.I_FOLDER, F.FOLDER_NAME, F.I_PROJECT, F.LEVEL_

  FROM FOLDERS__ F

    INNER JOIN FOLDERS_MAX__ M

        ON F.I_FOLDER = M.I_FOLDER

        AND F.LEVEL_ = M.LEVEL_

)

, TABLES_ AS (

SELECT MF.MOD_FOLDER_PATH, T.*

  FROM DEV_ODI_REPO.SNP_TABLE T

    INNER JOIN DEV_ODI_REPO.SNP_MODEL SM

        ON T.I_MOD = SM.I_MOD

    LEFT JOIN MOD_FOLDERS_ MF

        ON MF.I_MOD_FOLDER = SM.I_MOD_FOLDER

)

, MAPPINGS_ AS (

SELECT F.FOLDER_PATH, SM.*

  FROM DEV_ODI_REPO.SNP_MAPPING SM

    INNER JOIN FOLDERS_ F

        ON F.I_FOLDER = SM.I_FOLDER

WHERE 1 = 1

)

, COMPLETE_ AS (

SELECT DISTINCT SM.FOLDER_PATH

     , SM.NAME AS MAPPING_NAME

     , MC.NAME AS OBJECT_ALIAS

     , MR.QUALIFIED_NAME AS OBJECT_NAME

     , T.I_TABLE

     , T.MOD_FOLDER_PATH

     , SM.I_MAPPING

     , CP.I_MAP_CP

     , MR.I_REF_ID

     , CASE WHEN T.I_TABLE IS NULL THEN M.I_MAPPING ELSE NULL END AS REUSABLE_I_MAPPING

  FROM DEV_ODI_REPO.SNP_MAP_COMP MC

    INNER JOIN MAPPINGS_ SM

        ON MC.I_OWNER_MAPPING = SM.I_MAPPING

    INNER JOIN DEV_ODI_REPO.SNP_MAP_CP CP

        ON MC.I_MAP_COMP = CP.I_OWNER_MAP_COMP

    INNER JOIN DEV_ODI_REPO.SNP_MAP_REF MR

        ON MC.I_MAP_REF = MR.I_MAP_REF

    LEFT JOIN TABLES_ T

        ON MR.I_REF_ID = T.I_TABLE

    LEFT JOIN DEV_ODI_REPO.SNP_MAPPING M

        ON MR.I_REF_ID = M.I_MAPPING

WHERE 1 = 1

   AND CP.DIRECTION = 'O' --output

ORDER BY 1, 3

)

, SOURCES_ AS (

SELECT

       C.OBJECT_NAME AS SOURCE_OBJECT

     , C.*

  FROM COMPLETE_ C

WHERE C.I_MAP_CP IN (SELECT I_START_MAP_CP FROM DEV_ODI_REPO.SNP_MAP_CONN) --takes the target

)

, TARGETS_ AS (

SELECT C.OBJECT_NAME AS TARGET_TABLE

     , C.I_MAPPING

  FROM COMPLETE_ C

WHERE C.I_MAP_CP NOT IN (SELECT I_START_MAP_CP FROM DEV_ODI_REPO.SNP_MAP_CONN) --takes the target

)

, ALL_ AS (

SELECT T.TARGET_TABLE, S.*

  FROM SOURCES_ S

    INNER JOIN TARGETS_ T

        ON S.I_MAPPING = T.I_MAPPING

)

SELECT DISTINCT A.FOLDER_PATH, A.MAPPING_NAME, A.TARGET_TABLE

      , A.SOURCE_OBJECT, A.OBJECT_ALIAS, A.I_MAPPING, A.I_TABLE, A.REUSABLE_I_MAPPING, A.MOD_FOLDER_PATH

   FROM ALL_ A

  WHERE 1 = 1

   AND A.MAPPING_NAME = 'TEST2'

  ORDER BY 1, 2, 3, 4, 5;

ODI work repo - execution control (SLT).sql

-- EXECUCIONS
select 
       SS.SESS_STATUS status
     , DECODE(SS.SESS_STATUS, 'D', 'Done', 'E', 'Error', 'M', 'Warning', 'Q', 'Queued', 'R', 'Running', 'W', 'Waiting', SS.SESS_STATUS) AS status_d
     , SS.SESS_NO
     , SS.SCEN_NAME
     , SS.SCEN_VERSION version
     , SS.SESS_NAME
     , SS.PARENT_SESS_NO parent_sess
     , REGEXP_SUBSTR(ss.startup_variables, '(DATA_INI\=\d+)',1,1,'in',0) DATA_INI     
     , REGEXP_SUBSTR(ss.startup_variables, '(DATA_FIN\=\d+)',1,1,'in',0) DATA_FIN 
    , REGEXP_SUBSTR(ss.startup_variables, '(SYSDATE_1\=\d+)',1,1,'in',0) SYSDATE_1 
--    , REGEXP_SUBSTR(ss.startup_variables, '(DATA_CARREGA\=\d+)',1,1,'in',0) DATA_CARREGA 
     --, ss.startup_variables
     , SS.SESS_BEG
     , SS.SESS_END
     , SS.NB_ROW
     , SS.NB_INS
     , SS.NB_UPD
     , SS.NB_DEL
     , SS.NB_ERR     
FROM  ETQPRO_BISLT_REPO.SNP_SESSION SS 
where 1=1 
    /* filtre temporal */
    AND SS.SESS_BEG >= TO_DATE('20201010','yyyymmdd') and SS.SESS_BEG < TO_DATE('20201014','yyyymmdd')+1
    --AND SS.SESS_BEG >= TRUNC(SYSDATE) - 3  /* ex. cap setmana */
    --AND SS.SESS_BEG >= TRUNC(SYSDATE)
 /* filtre errors i warnings */   
    and (SS.SESS_STATUS = 'E'
        OR (SS.SESS_STATUS = 'M' AND SS.NB_ERR > 0) 
    )
    /* filtre escenaris concrets */
    --and SS.SCEN_NAME <> 'WF_DDS_SIMD_30_STUDY_UPD_IMG_PUBLI'    

 ORDER BY SS.SESS_STATUS ASC, SS.SCEN_NAME ASC, SS.SESS_BEG ASC;



 
--select  distinct owner from all_tables;
-- https://community.oracle.com/thread/4279794
-- RUNTIME
SELECT SS.SESS_NO
     , SS.SCEN_NAME
     , SS.SCEN_VERSION
--     , SS.SESS_NAME
--     , SS.SESS_BEG
     --, SS.SESS_END
--     , SS.SESS_STATUS
     , DECODE(SS.SESS_STATUS, 'D', 'Done', 'E', 'Error', 'M', 'Warning', 'Q', 'Queued', 'R', 'Running', 'W', 'Waiting', SS.SESS_STATUS) AS SESS_STATUS_DESC

     -- detalls del pas dins el package (cada mapping, etc.)
     , SSS.STEP_TYPE
     , sss.step_name     
     , SST.TASK_NAME1
     , SST.TASK_NAME2
     , SST.TASK_NAME3
     , SSTL.TASK_DUR
--     , ssl.step_status
     , DECODE(ssl.step_status, 'D', 'Done', 'E', 'Error', 'M', 'Warning', 'Q', 'Queued', 'R', 'Running', 'W', 'Waiting', ssl.step_status) AS STEP_STATUS_DESC
     , ssl.step_beg
--    , ssl.step_end
    , ssl.step_dur
--     , SSTL.NB_ROW task_row
--     , SSTL.NB_INS task_ins
--     , SSTL.NB_UPD task_upd
--     , SSTL.NB_DEL task_del
     , SSTL.NB_ERR tasl_err
--     , SSL.NB_ROW step_row
--     , SSL.NB_INS step_ins
--     , SSL.NB_UPD step_upd
--     , SSL.NB_DEL step_del
     , SSL.NB_ERR step_err
     , ssl.error_message



     , SSS.LSCHEMA_NAME || '.' || SSS.RES_NAME AS TARGET_TABLE
     , CASE WHEN SST.COL_TECH_INT_NAME IS NOT NULL AND SST.COL_LSCHEMA_NAME IS NOT NULL THEN SST.COL_TECH_INT_NAME || '.' || SST.COL_LSCHEMA_NAME
            ELSE NULL
       END AS TARGET_SCHEMA
     , SSTL.DEF_TXT AS TARGET_COMMAND
     , CASE WHEN SST.DEF_TECH_INT_NAME IS NOT NULL AND SST.DEF_LSCHEMA_NAME IS NOT NULL THEN SST.DEF_TECH_INT_NAME || '.' || SST.DEF_LSCHEMA_NAME
            ELSE NULL
       END AS SOURCE_SCHEMA
     , SSTL.COL_TXT AS SOURCE_COMMAND

--     , SSTL.NB_RUN
--     , SST.TASK_TYPE
     , DECODE(SST.TASK_TYPE, 'C', 'Loading', 'J', 'Mapping', 'S', 'Procedure', 'V', 'Variable', SST.TASK_TYPE) AS TASK_TYPE_DESC
--     , SST.EXE_CHANNEL
--     , DECODE(SST.EXE_CHANNEL, 'B', 'Oracle Data Integrator Scripting', 'C', 'Oracle Data Integrator Connector', 'J', 'JDBC', 'O', 'Operating System', 'Q', 'Queue', 'S', 'Oracle Data Integrator Command', 'T', 'Topic', 'U', 'XML Topic', SST.EXE_CHANNEL) AS EXE_CHANNEL_DESC

     , SS.PARENT_SESS_NO
     , SSL.NNO stepLog_no     
     , SSTL.SCEN_TASK_NO
     , SST.PAR_SCEN_TASK_NO
     , SSS.SB_NO
     , SSS.NNO sbStep_num  -- es igual que SSL.NNO     


  FROM ETQPRO_BISLT_REPO.SNP_SESSION SS
    INNER JOIN ETQPRO_BISLT_REPO.SNP_STEP_LOG SSL
        ON SS.SESS_NO = SSL.SESS_NO
    INNER JOIN ETQPRO_BISLT_REPO.SNP_SESS_TASK_LOG SSTL
        ON SS.SESS_NO = SSTL.SESS_NO
    INNER JOIN ETQPRO_BISLT_REPO.SNP_SB_TASK SST
        ON SSTL.SB_NO = SST.SB_NO
        AND SSTL.SCEN_TASK_NO = SST.SCEN_TASK_NO
        AND SSL.NNO = SSTL.NNO
        AND SSTL.NNO = SST.NNO
        AND SSL.NB_RUN = SSTL.NB_RUN
    LEFT JOIN ETQPRO_BISLT_REPO.SNP_SB_STEP SSS
        ON SST.SB_NO = SSS.SB_NO
        AND SST.NNO = SSS.NNO
WHERE 1 = 1
--   AND SS.SESS_BEG > TO_DATE('20200907','yyyymmdd') -- trunc(sysdate)
   AND SS.SESS_BEG > trunc(sysdate)
   -- AND SS.SESS_NO = 123542
--   AND SST.EXE_CHANNEL IN ('B', 'C', 'J', 'S')
--   AND (UPPER(SSTL.DEF_TXT) LIKE '%SELECT%' OR UPPER(SSTL.DEF_TXT) LIKE '%MERGE%' OR UPPER(SSTL.DEF_TXT) LIKE '%INSERT%')
   AND SST.TASK_TYPE NOT IN ('V') --VARIABLE
   
   -- AND SS.SESS_STATUS IN ('D', 'R', 'M', 'E')   
    and (
        (SS.SESS_STATUS = 'E' and SSL.step_status = 'E' )
        OR 
        (SS.SESS_STATUS = 'M' AND SSTL.NB_ERR > 0)
    )
    
    -- filtres sobre els passos
    --AND SSS.STEP_TYPE = 'M' -- M mapping,  VC variable
    --AND SSS.NNO = 2 -- num del pas dins el package (pot ser declaració variable, mapping, ...)
    
ORDER BY SS.SESS_BEG, ssl.step_beg, SSS.SB_NO, SSS.NNO;

ODI work repository - search expressions.sql

-- EXPRESSIONS
-- Retorna una fila per cada variable que conté l'expressió:
select me.I_MAP_EXPR, me.I_OWNER_MAP_ATTR, me.PARSED_TXT
, mer.REF_KEY, mer.REF_TEXT
, me.TXT
, mer.I_MAP_EXPR_REF, mer.I_REF_MAP_ATTR
, me.I_MAP_CP, mer.I_OWNER_MAP_EXPR
FROM ETQINT_BISLT_REPO.SNP_MAP_EXPR me 
inner join ETQINT_BISLT_REPO.SNP_MAP_EXPR_REF mer on (mer.I_OWNER_MAP_EXPR = me.I_MAP_EXPR)
where 1=1
--and me.TXT like '%CPA_IDP%' -- aquest camp conté tota l'expressió
and mer.REF_TEXT like '%CPA_IDP%' -- columna sobre la que aplica l'expressió
and me.PARSED_TXT like '%null%' -- paraules clau de l'expressió, sense incloure el nom de la columna
;

-- o cercant directament a l 'expressió sense parsejar:
select me.I_MAP_EXPR, me.I_OWNER_MAP_ATTR, me.PARSED_TXT, me.TXT , me.I_MAP_CP
FROM ETQINT_BISLT_REPO.SNP_MAP_EXPR me 
where me.TXT like '%CPA_IDP%' -- aquest camp conté tota l'expressió
;


WITH base as (
    select me.I_MAP_EXPR, me.TXT exp_txt, mc.name comp_name, m.NAME map_name
    from ETQINT_BISLT_REPO.snp_mapping m 
    inner join ETQINT_BISLT_REPO.snp_map_comp mc on m.i_mapping = mc.i_owner_mapping
    inner join ETQINT_BISLT_REPO.snp_map_cp cp on mc.i_map_comp = cp.i_owner_map_comp
    inner join ETQINT_BISLT_REPO.SNP_MAP_EXPR me on me.I_MAP_CP = cp.I_MAP_CP
    UNION ALL 
    select me.I_MAP_EXPR, me.TXT exp_txt, mc.name comp_name, m.NAME map_name
    from ETQINT_BISLT_REPO.snp_mapping m 
    inner join ETQINT_BISLT_REPO.snp_map_comp mc on m.i_mapping = mc.i_owner_mapping
    inner join ETQINT_BISLT_REPO.SNP_MAP_PROP mp on mc.i_map_comp = mp.i_owner_map_comp
    inner join ETQINT_BISLT_REPO.SNP_MAP_EXPR me on mp.I_MAP_PROP = me.I_OWNER_MAP_PROP
)
select * FROM base
where I_MAP_EXPR IN (41132,41111,43678,41213,41186,43697,42551,45559,45561,45588,46308,43963)
;

ODI-Repository-queries.sql

-- https://www.clearpeaks.com/odi-11g-repository-metadata-tips-tricks/
-- https://www.clearpeaks.com/oracle-data-integrator-11g-ii-data-lineage/
-- https://sonra.io/2012/09/17/get-a-list-of-all-of-your-odi-objects-including-the-path-in-your-project-or-model/


-- Para estudiar el repo:
select * FROM USER_TABLES;
select DISTINCT owner FROM ALL_TABLES;
-- ETQINT_BISLT_REPO
-- ETQINT_BISLT2_REPO

-- Master Repo (y algunos Work):
SELECT * FROM SNP_AGENT_PROP_LST;
SELECT * FROM SNP_ALLOC_AGENT;
SELECT * FROM SNP_CONNECT;
SELECT * FROM SNP_CONNECT_PROP;
SELECT * FROM SNP_CONTEXT;
SELECT * FROM SNP_INST_OBJ;
SELECT * FROM SNP_LAGENT;
SELECT * FROM SNP_LOC_REP;
SELECT * FROM SNP_LOC_REPW;
SELECT * FROM SNP_LSCHEMA;
SELECT * FROM SNP_PSCHEMA;
SELECT * FROM SNP_PSCHEMA_CONT;
SELECT * FROM SNP_USER;

-- Work Repo
SELECT * FROM SNP_ACTION;
SELECT * FROM SNP_AGENT;
SELECT * FROM SNP_CONV_DT;
SELECT * FROM SNP_DATA;
SELECT * FROM SNP_DT;
SELECT * FROM SNP_ENT_ID;
SELECT * FROM SNP_ENTITY;
SELECT * FROM SNP_FIELD;
SELECT * FROM SNP_FIELD_LOOKUP;
SELECT * FROM SNP_FLEX_FIELD;
SELECT * FROM SNP_GRP_ACTION;
SELECT * FROM SNP_IMPORT_REP;
SELECT * FROM SNP_INDEX_TYPE;

SELECT * FROM SNP_LANG;
SELECT * FROM SNP_LANG_ELT;
SELECT * FROM SNP_LANG_TECHNO;
SELECT * FROM SNP_LE_TECHNO;
SELECT * FROM SNP_LINE_ACTION;
SELECT * FROM SNP_LINE_TRT;
SELECT * FROM SNP_LINE_TRT_UE;
SELECT * FROM SNP_LINK;

SELECT * FROM SNP_MAP_COMP_TYPE;
SELECT * FROM SNP_MAP_CP_ROLE;
SELECT * FROM SNP_MAP_PROP_DEF;
SELECT * FROM SNP_METHOD;
SELECT * FROM SNP_M_IMPORT_REP;
SELECT * FROM SNP_MODULE;
SELECT * FROM SNP_MTXT;
SELECT * FROM SNP_OBJECT;
SELECT * FROM SNP_PROFILE;
SELECT * FROM SNP_PROF_METH;

SELECT * FROM SNP_PWD_POLICY;
SELECT * FROM SNP_PWD_RULE;
SELECT * FROM SNP_REM_REP;
SELECT * FROM SNP_SUB_LANG;
SELECT * FROM SNP_TECHNO;
SELECT * FROM SNP_TRT;
SELECT * FROM SNP_TXT_HEADER;
SELECT * FROM SNP_U_O_M_REP;

SELECT * FROM SNP_USER_EXIT;
SELECT * FROM SNP_USER_METH;
SELECT * FROM SNP_USER_OBJ_METH;
SELECT * FROM SNP_USER_PROF;
SELECT * FROM SNP_USR_PREFERENCE;
SELECT * FROM SNP_VERSION;


Mappings
=======
-- adaptada de https://www.rittmanmead.com/blog/2016/02/di-tips-odi12c-repo-query-mapping-target-table/
select
    m.name mapping_name,
    mr.qualified_name,
    mc.name datastore_alias,
    t.table_name target_table,
    mdl.cod_mod model_code
from ETQINT_BISLT_REPO.snp_mapping m 
    inner join ETQINT_BISLT_REPO.snp_map_comp mc on m.i_mapping = mc.i_owner_mapping
    inner join ETQINT_BISLT_REPO.snp_map_cp cp on mc.i_map_comp = cp.i_owner_map_comp
    inner join ETQINT_BISLT_REPO.snp_map_ref mr on mc.i_map_ref = mr.i_map_ref
    inner join ETQINT_BISLT_REPO.snp_table t on mr.i_ref_id = t.i_table
    inner join ETQINT_BISLT_REPO.snp_model mdl on t.i_mod = mdl.i_mod
where cp.direction = 'O' and --output connection point
    cp.i_map_cp not in
        (select i_start_map_cp from ETQINT_BISLT_REPO.snp_map_conn) --not a starting connection point
        
--    and m.name like '%CCC%'
    and t.table_name in ('FT_HC3_CCC', 'FT_HC3_CCC_DELTA', 'FT_HC3_APAT_HC3TBAPT_DELTA', 'FT_HC3_APAT_HC3TBAPT', 'D_HC3_DIAGNOSTICS')
order by mdl.cod_mod, t.table_name, m.name
;



SCENARIOS
=========
select *
from SNP_SCEN
where 1=1
  and SCEN_NAME like '%SDE%'
order by SCEN_NAME;44


-- Confirm whether some scenarios have been regenerated after the corresponding packages/steps/interfaces (SNP_PACKAGE, SNP_STEP) 
-- have been modified. This can be achieved by comparing the last modification dates of the objects, as shown below:
select *
from (
  select distinct
    SCEN.SCEN_NAME, SCEN.LAST_DATE SCEN_LAST_DT,
    PACK.PACK_NAME, PACK.LAST_DATE PACK_LAST_DT,
    STPOP.POP_NAME STEP_POP_NAME, STPOP.LAST_DATE STEP_POP_LAST_DT,
    POP.POP_NAME, POP.LAST_DATE POP_LAST_DT,
    case when (
      SCEN.LAST_DATE > coalesce(PACK.LAST_DATE, to_date('20000101','yyyymmdd')) and 
      SCEN.LAST_DATE > coalesce(STPOP.LAST_DATE, to_date('20000101','yyyymmdd')) and 
      SCEN.LAST_DATE > coalesce(POP.LAST_DATE, to_date('20000101','yyyymmdd')) 
    ) then 'Y' else 'N' end REGENERATED_FLG
  from SNP_SCEN SCEN
  left outer join SNP_PACKAGE PACK on SCEN.I_PACKAGE = PACK.I_PACKAGE
  left outer join SNP_STEP STEP on PACK.I_PACKAGE = STEP.I_PACKAGE
  left outer join SNP_POP STPOP on STEP.I_POP = STPOP.I_POP
  left outer join SNP_POP POP on SCEN.I_POP = POP.I_POP
  where 1=1
    and (STEP.I_POP is not null or SCEN.I_POP is not null)
)
where 1=1
  and REGENERATED_FLG = 'N'
order by SCEN_NAME, PACK_NAME, STEP_POP_NAME, POP_NAME;



DATA LINEAGE
============
-- https://www.clearpeaks.com/oracle-data-integrator-11g-ii-data-lineage/
                                                            
--Let’s just begin with the previously mentioned query and discuss some of its parts. 
-- This query is at the lowest level possible and describes the Sources and Target, and Mappings between the Columns 
-- for each Interface. Removing some of these query columns would give a more general and simple overview, 
-- like the relation between Interfaces, Sources and Target tables without the Columns level.

select distinct 
 
  -- FOLDERS : more levels can be included, if needed (update the join conditions accordingly)
  PROJECT.PROJECT_NAME PROJECT,
  FOLDER_LVL1.FOLDER_NAME LVL1_FOLDER,
  FOLDER_LVL2.FOLDER_NAME LVL2_FOLDER,
   
  --INTERFACE
  I.POP_NAME INTERFACE,
  case when I.WSTAGE = 'E' then 'N' else 'Y' end TEMPORARY_INTERFACE, -- WSTAGE can be: E - Existing target or N,W - Temporary target
   
  -- SOURCE
  SRC_TMP.POP_NAME SOURCE_TMP, -- temporary interface as the source
  SRC_MOD.MOD_NAME SOURCE_MODEL,
  SRC_TAB.TABLE_NAME SOURCE_TABLE,
  SRC_TAB.RES_NAME SOURCE_TABLE_RES_NAME,
  SRC_COL.COL_NAME SOURCE_COLUMN,
  SRC_COL.SOURCE_DT || '(' || SRC_COL.LONGC || case when SRC_COL.SCALEC is not null then ',' || SRC_COL.SCALEC else '' end || ')' SOURCE_COLUMN_DATATYPE,
   
  -- TARGET
  TGT_MOD.MOD_NAME TARGET_MODEL,   
  case when I.WSTAGE = 'E' then TGT_TAB.TABLE_NAME || '(' || TGT_TAB.RES_NAME || ')' else I.TABLE_NAME end TARGET_TABLE,
  case when I.WSTAGE = 'E' then TGT_TAB.RES_NAME else null end TARGET_TABLE_RES_NAME,
  case when I.WSTAGE = 'E' then TGT_COL.COL_NAME else TGT_POP_COL.COL_NAME end TARGET_COLUMN,
  case
    when I.WSTAGE = 'E' then TGT_COL.SOURCE_DT || '(' || TGT_COL.LONGC || case when TGT_COL.SCALEC is not null then ',' || TGT_COL.SCALEC else '' end || ')'
    else TGT_POP_COL.SOURCE_DT || '(' || TGT_POP_COL.LONGC || case when TGT_POP_COL.SCALEC is not null then ',' || TGT_POP_COL.SCALEC else '' end || ')'
  end TARGET_COLUMN_DATATYPE,
  case when TGT_POP_COL.IND_KEY_UPD = 1 then 'Y' else null end PRIMARY_KEY,
   
  -- MAPPING VALUE
  MAP_VAL_FULL.STRING_ELT FULL_ELT_STRING
 
from SNP_PROJECT PROJECT
  left outer join SNP_FOLDER FOLDER_LVL1 on FOLDER_LVL1.I_PROJECT = PROJECT.I_PROJECT -- FIRST FOLDER LEVEL
  left outer join SNP_FOLDER FOLDER_LVL2 on FOLDER_LVL2.PAR_I_FOLDER = FOLDER_LVL1.I_FOLDER -- SECOND FOLDER LEVEL
 
  left outer join SNP_POP I on I.I_FOLDER = FOLDER_LVL2.I_FOLDER -- INTERFACES IN THE 2nd LEVEL FOLDER
 
  left outer join SNP_POP_COL TGT_POP_COL on TGT_POP_COL.I_POP = I.I_POP -- TARGET COLUMNS OF THE INTERFACES
  left outer join SNP_COL TGT_COL on TGT_COL.I_COL = TGT_POP_COL.I_COL -- TARGET COLUMNS DETAILS
  left outer join SNP_TABLE TGT_TAB on TGT_TAB.I_TABLE = TGT_COL.I_TABLE -- TARGET TABLE DETAILS
  left outer join SNP_MODEL TGT_MOD on TGT_MOD.I_MOD = TGT_TAB.I_MOD -- TARGET TABLE MODEL
 
  left outer join SNP_POP_MAPPING MAP on MAP.I_POP_COL = TGT_POP_COL.I_POP_COL 
  left outer join SNP_TXT_CROSSR MAP_VAL on MAP_VAL.I_TXT = MAP.I_TXT_MAP and MAP_VAL.OBJECT_TYPE in ('C', 'P', 'V') 
  left outer join SNP_COL SRC_COL on SRC_COL.I_COL = MAP_VAL.I_COL -- SOURCE COLUMN DETAILS
  left outer join SNP_TABLE SRC_TAB on SRC_TAB.I_TABLE = SRC_COL.I_TABLE -- SOURCE TABLE DETAILS
  left outer join SNP_MODEL SRC_MOD on SRC_MOD.I_MOD = SRC_TAB.I_MOD -- SOURCE TABLE MODEL
  left outer join SNP_DATA_SET DATA_SET on I.I_POP = DATA_SET.I_POP 
  left outer join SNP_SOURCE_TAB SOURCE_TAB on DATA_SET.I_DATA_SET = SOURCE_TAB.I_DATA_SET
  left outer join SNP_POP SRC_TMP on SOURCE_TAB.I_POP_SUB = SRC_TMP.I_POP -- TEMPORARY SOURCE
 
  left outer join (
    select I_TXT, STRING_POS, STRING_ELT, ROW_NUMBER() over (partition by I_TXT order by length(STRING_ELT) desc) POS 
    from SNP_TXT_CROSSR 
   ) MAP_VAL_FULL on MAP_VAL_FULL.I_TXT = MAP_VAL.I_TXT and MAP_VAL_FULL.POS = 1
 
where 1=1 -- FILTERS
  and PROJECT.PROJECT_NAME = 'BI Apps Project' -- by Project name
  and FOLDER_LVL1.FOLDER_NAME like 'Custom_%' -- by 1st Level Folder name
  and FOLDER_LVL2.FOLDER_NAME = 'SDE_ORA_%' -- by 2nd Level Folder name
  and I.POP_NAME not like 'Copy%' -- by Interface name
  and SRC_TAB.TABLE_NAME like '%WC_LHA%PS%' -- by Source Table name
  and SRC_COL.COL_NAME like '%EMPLOYEE_ID' -- by Source Column name
  and TGT_TAB.TABLE_NAME like '%_DS' -- by Target Table name
  and TGT_COL.COL_NAME like '%EMPLOYEE_ID'  -- by Target Column name
  and MAP_VAL_FULL.STRING_ELT like '%COALESCE(%' -- by Mapping Value
         
order by 
  PROJECT.PROJECT_NAME
  , FOLDER_LVL1.FOLDER_NAME
  , FOLDER_LVL2.FOLDER_NAME 
  , I.POP_NAME 
  , SRC_COL.COL_NAME
  , case when I.WSTAGE = 'E' then TGT_COL.COL_NAME else TGT_POP_COL.COL_NAME end
;



-- IMPORTANTE: Recordar ejecutar en el usuario del Work Repository, no del Master Repository (a menos que queramos ver sólo los KM, etc.)


-- https://sonra.io/2012/09/17/get-a-list-of-all-of-your-odi-objects-including-the-path-in-your-project-or-model/
WITH obj as(
SELECT i_package i_instance, i_folder, NULL i_project, pack_name obj_name,3200 obj_type, 'Package' obj_type_name, last_date, last_user FROM SNP_PACKAGE
 UNION ALL
SELECT i_pop i_instance, i_folder, NULL i_project, pop_name obj_name,3100 obj_type, 'Interface' obj_type_name, last_date, last_user FROM SNP_POP
 UNION ALL
SELECT i_trt i_instance, i_folder, i_project, trt_name obj_name,3600 obj_type, CASE trt_type WHEN 'U' THEN 'Procedure' ELSE 'Knowledge Module' END obj_type_name, last_date, last_user FROM SNP_TRT
 UNION ALL
SELECT i_var i_instance,  NULL i_folder, i_project, var_name obj_name, 3500 obj_type, 'Variable' obj_type_name, last_date, last_user FROM SNP_VAR t
 UNION ALL
SELECT i_table i_instance, i_sub_model i_folder, i_mod i_project, table_name obj_name, 2400 obj_type, 'Table' obj_type_name, last_date, last_user from snp_table t
)
,fd (i_folder, i_project, folder_name, folder_path, lv) AS(
SELECT i_folder, i_project, folder_name, folder_name folder_path, 1 lv
  FROM snp_folder
 WHERE par_i_folder IS NULL
 UNION ALL
SELECT tf.i_folder, tf.i_project, tf.folder_name, fd.folder_path||''||tf.folder_name, fd.lv+1
  FROM snp_folder tf JOIN fd
    ON fd.i_folder = tf.par_i_folder
)
,mpl as (
SELECT sm.i_smod i_mc, 'sm' typemc, COALESCE(sm.i_smod_parent,sm.i_mod) i_mp, NVL2(sm.i_smod_parent,'sm','m') typemp, sm.smod_name name --, m.i_mod_folder
  FROM snp_sub_model sm
 UNION ALL
SELECT i_mod, 'm' typ, i_mod_folder, 'mf', mod_name
  FROM snp_model m
 UNION ALL
SELECT i_mod_folder, 'mf', par_i_mod_folder, 'mf', mod_folder_name FROM snp_mod_folder
)
,mp (i_mc, typemc, i_mp, typemp, model_tech, model_path, lv) AS(
SELECT i_mc, typemc, i_mp, typemp, name tname, name model_path, 1 lv
  FROM mpl
 WHERE i_mp IS NULL
 UNION ALL
SELECT mpl.i_mc, mpl.typemc, mpl.i_mp, mpl.typemp, mp.model_tech, mp.model_path||''||mpl.name model_path, mp.lv+1 lv
  FROM mpl JOIN mp
    ON mpl.i_mp = mp.i_mc AND mpl.typemp=mp.typemc
)
SELECT obj.i_instance, OBJ_NAME, CASE WHEN COALESCE(project_name,mp.model_tech) IS NULL THEN 'Global ' || OBJ_TYPE_NAME ELSE OBJ_TYPE_NAME END obj_type_name
      ,obj.last_date
      ,obj.last_user
      ,COALESCE(project_name,mp.model_tech) project_model
      ,COALESCE(fd.folder_path,mp.model_path) path
  FROM obj
  LEFT OUTER
  JOIN fd
    ON fd.i_folder = obj.i_folder AND obj_type_name!='Table'
  LEFT OUTER
  JOIN mp
    ON mp.i_mc = obj.i_folder AND obj_type_name='Table'
  LEFT OUTER
  JOIN snp_project p
    ON p.i_project = COALESCE(obj.i_project, fd.i_project)
  LEFT OUTER
  JOIN snp_model m
    ON m.i_mod = obj.i_project
    ;

Oracle Data Integrator ODI - compare scenario versions using SQL

-- https://www.linkedin.com/pulse/comparing-odi-scenario-versions-using-sql-ricardo-giampaoli
-- https://devepm.com/2017/10/26/comparing-odi-scenario-versions-using-sql/


Comparing ODI Scenario versions using SQL
==========================================


Hi all, it has being a while that we don’t post! Busy days you know…. Anyway, let’s see what we got today.

The situation that I’m about to describe often happens in large/old ODI projects. Imagine the following: you received a task to change an ODI component that was created one year ago by someone else that is not even in the company anymore. The code is running fine in PROD and the business want a small fix to it. You open the ODI package and it contains a lot of interfaces, procedures, variables, etc. You need to change the code in one single interface, which seems very simple. You change it, save it, generate a new scenario and move it to PROD. When it gets there, the job fails due to an error in another interface that you did not touch! You start to troubleshoot and figure out that someone else changed something in DEV, saved it, but did not move the code to PROD. Unfortunately this ”unwanted” code change was included by you when you generated your scenario and now the mess is already created. If you already passed through this situation, than this post may help you.

Code versioning and code migration processes in general are things that everybody knows that are necessary, but sometimes they are overlooked by the companies because people think they are too complicated or does not work very well. ODI is a big example of this, since its native versioning system is not very intuitive and most of the times does not work in the way that we want to. There are companies out there that even build their own code versioning system (outside of ODI) to manage ODI code versions. I dare to say that most of the companies don’t even have any kind of code versioning or formal code migration process for ODI at all, which causes some big headaches on similar situations as the one that I just described.

The technique that I’ll explain here is not about code versioning itself. I’ll describe something that we may use when we do not have any other way to guarantee that the scenario that we are generating was not changed by someone else during a time period. Just to let you know, all the following SQL was done in ODI 11.1.1.9 version.

Let’s begin with the basics. Everything that you create in ODI is stored in SNP tables in its WORK and MASTER repositories. For this post we will focus in two main tables from the WORK repository:

SNP_SCEN: contains the basic information about the scenarios that exists in that WORK repository (like name, version, creation date and so on);
SNP_SCEN_TASK: the “main” scenario table that contains all the steps/tasks that are performed by a scenario. You may query this table and see exactly which tasks (like SQL commands, variables, flows) that scenario will perform when you run it in Operator;
So now let’s get back to our problem. There is a scenario that is running fine in Production for one year now (let’s say it calls ODI_SCENARIO Version 1_00_00) and this scenario is also in Development. I’ll make a change in only one interface (I’ll add a simple SUBSTR in a column named PACK_SLIP) of this scenario in Development and create a new version of it (ODI_SCENARIO Version 2_00_00). How do I guarantee that my code change was the only thing that changed in this new scenario and that it does not contain any other code from other developers? The answer lies on the SNP_SCEN_TASK table.

If you go to ODI WORK repository in Production, you may run the following query to get all the steps that the scenario is currently executing on its 1_00_00 Version:

SELECT NNO,
SCEN_TASK_NO,
TASK_TYPE,
TASK_NAME1,
TASK_NAME2,
TASK_NAME3,
EXE_CHANNEL,
DEF_CONTEXT_CODE,
DEF_LSCHEMA_NAME,
DEF_CONNECT_ID,
DEF_IND_COMMIT,
DEF_ISOL_LEVEL,
DEF_PLAN_COMP,
COL_CONTEXT_CODE,
COL_LSCHEMA_NAME,
COL_CONNECT_ID,
COL_ISOL_LEVEL,
COL_IND_COMMIT,
COL_PLAN_COMP,
ORD_TRT,
IND_ERR,
LOG_LEV_DET,
IND_LOG_NB,
DEF_TECH_INT_NAME,
COL_TECH_INT_NAME,
IND_LOG_METHOD,
COL_TXT,
COL_IND_ENC,
COL_ENC_KEY,
DEF_TXT,
DEF_IND_ENC,
DEF_ENC_KEY,
IND_LOG_FINAL_CMD
FROM SNP_SCEN_TASK
WHERE SCEN_NO IN
(SELECT SCEN_NO
FROM SNP_SCEN
WHERE SCEN_NAME = 'ODI_SCENARIO'
AND SCEN_VERSION = '1_00_00')
ORDER BY SCEN_TASK_NO,NNO;

2017-10-25_17-17-51

There are a lot of important columns in this table that can give you a lot of valuable information. However, COL_TXT and DEF_TXT are generally the most important ones since they contain the code that is generated in the “Source and Target tabs” inside the procedures and interfaces. After you run this SQL in Production environment, you may export it to whatever you like. In this example here, I’ll export it as “Text” using Oracle SQL Developer as the following (Right click on any row and select “Export”):

2017-10-25_17-21-19

Save it somewhere in your computer:

2017-10-25_17-22-04

The result will be something like this:

2017-10-25_17-24-01

Now let’s run the SQL in the Development ODI WORK repository. The only thing that we will change now is our filter that will go from SCEN_VERSION = ‘1_00_00’ to SCEN_VERSION = ‘2_00_00’, which is the new scenario version that we just generated. Do the same steps as the Production SQL and you should end up with something like this:

2017-10-25_17-27-19

Now you need to compare both codes. I like to go simple and use Notepad ++ with “Compare” plugin. You may use any other tool for comparing txt files (Beyond Compare is awesome as well). In Notepad ++ you just need to open both files, click on the Production file and “Set the First Compare”, then click on de Development file and “Compare”.

2017-10-25_17-31-42

2017-10-25_17-32-54

You will have something similar to this when you compare:

2017-10-25_17-34-24

The “Compare NavBar” shows a lot of differences, way more than the one that I just did. However, we need to analyze it calmly to verify what do they really mean. You may navigate thought the changes using the “Next” button in the tool bar.

2017-10-25_17-43-00

There will be some blocks of code that contains “similar differences” due to the nature of ODI. For example, when you change one single thing in one column of one interface, it will be reflected in several steps within the Knowledge Module (in C$/I$/E$ creation for example). This is one example of it:

2017-10-25_17-49-46

This change is saying that we changed the order of PACK_SLIP column (which was the column that we added a SUBSTR command). Actually we didn’t change the order, but we changed its content. However, when ODI create its temporary tables (like C$, I$ and E$) we cannot control the order that they are going to be created as the code is generated automatically by ODI. So we don’t need to worry about this change, as it was somehow “expected”. When we click “Next”, we are going to have similar ones where the column just changed its order. Continuing further down, we will get to the place where our change occurred:

2017-10-25_17-58-43

Cool, this is the place that we changed our code and it looks good. Let’s keep going to see what else has changed. Now we will get something weird. A lot of “1” changes just appeared until the end of the file (explaining why we had a lot of changes in the comparison Navigation Bar):

2017-10-25_17-59-56

This “1” comes from IND_LOG_FINAL_CMD column, which identifies if the step should “Log Final Command” or not. This does not affect the code itself, but for the sake of my analyses I went to the KM to see if someone had changed this option:

2017-10-25_18-06-42

My suspicious was right and someone changed this option in one of the KMs, which got reflected in a lot of places in my ODI scenario. There was no more changes in my comparison, so I could conclude that:

PACK_SLIP changed the order in some temporary tables creation, which is ok;
I saw my PACK_SLIP mapping change (SUBSTR) in the Development code;
There was a change in the KM to “Log Final Command” in a specific KM step, which is also ok and does not affect the code itself;
No more differences were found between the scenarios, so I may safely deploy it to production. If someone else had changed something more critical, the compare method would have catch that and we could revert it back before moving to Production.

There are other ways for you to get and compare the codes, like if both scenarios are in the same DB you could just run two SQLs and compare them or you could export both XML scenario files and compare those, but this post here gives you a generic way that can be done in most of the cases and it is fairly easy to be used.