Unified Audit Relink is looking for gcc after applying RU 19.29

Following error was observed while relinking for unified audit option after applying RU 19.29.

echo $ORACLE_HOME
/opt/app/oracle/product/19.29.0/dbhome_1

cd $ORACLE_HOME/rdbms/lib
make -f ins_rdbms.mk uniaud_on ioracle ORACLE_HOME=$ORACLE_HOME
/bin/sh: /usr/bin/gcc: No such file or directory
expr: missing operand
Try 'expr --help' for more information.
/bin/sh: line 0: test: -gt: unary operator expected
/usr/bin/ar cr /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/libknlopt.a /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/kzaiang.o
chmod 755 /opt/app/oracle/product/19.29.0/dbhome_1/bin
cd /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/;\
/usr/bin/ar r  /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/libknlopt.a `/usr/bin/ar t /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/libknlopt.a` ;

 - Linking Oracle
rm -f /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/oracle
/opt/app/oracle/product/19.29.0/dbhome_1/bin/orald  -o /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/oracle -m64 -z noexecstack -Wl,--disable-new-dtags -L/opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/ -L/opt/app/oracle/product/19.29.0/dbhome_1/lib/ -L/opt/app/oracle/product/19.29.0/dbhome_1/lib/stubs/   -Wl,-E /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/opimai.o /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/ssoraed.o /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/ttcsoi.o -Wl,--whole-archive -lperfsrv19 -Wl,--no-whole-archive /opt/app/oracle/product/19.29.0/dbhome_1/lib/nautab.o /opt/app/oracle/product/19.29.0/dbhome_1/lib/naeet.o /opt/app/oracle/product/19.29.0/dbhome_1/lib/naect.o /opt/app/oracle/product/19.29.0/dbhome_1/lib/naedhs.o /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/config.o  -ldmext -lserver19 -lodm19 -lofs -lcell19 -lnnet19 -lskgxp19 -lsnls19 -lnls19  -lcore19 -lsnls19 -lnls19 -lcore19 -lsnls19 -lnls19 -lxml19 -lcore19 -lunls19 -lsnls19 -lnls19 -lcore19 -lnls19 -lclient19  -lvsnst19 -lcommon19 -lgeneric19 -lknlopt -loraolap19 -lskjcx19 -lslax19 -lpls19  -lrt -lplp19 -ldmext -lserver19 -lclient19  -lvsnst19 -lcommon19 -lgeneric19 `if [ -f /opt/app/oracle/product/19.29.0/dbhome_1/lib/libavserver19.a ] ; then echo "-lavserver19" ; else echo "-lavstub19"; fi` `if [ -f /opt/app/oracle/product/19.29.0/dbhome_1/lib/libavclient19.a ] ; then echo "-lavclient19" ; fi` -lknlopt -lslax19 -lpls19  -lrt -lplp19 -ljavavm19 -lserver19  -lwwg  `cat /opt/app/oracle/product/19.29.0/dbhome_1/lib/ldflags`    -lncrypt19 -lnsgr19 -lnzjs19 -ln19 -lnl19 -lngsmshd19 -lnro19 `cat /opt/app/oracle/product/19.29.0/dbhome_1/lib/ldflags`    -lncrypt19 -lnsgr19 -lnzjs19 -ln19 -lnl19 -lngsmshd19 -lnnzst19 -lzt19 -lztkg19 -lmm -lsnls19 -lnls19  -lcore19 -lsnls19 -lnls19 -lcore19 -lsnls19 -lnls19 -lxml19 -lcore19 -lunls19 -lsnls19 -lnls19 -lcore19 -lnls19 -lztkg19 `cat /opt/app/oracle/product/19.29.0/dbhome_1/lib/ldflags`    -lncrypt19 -lnsgr19 -lnzjs19 -ln19 -lnl19 -lngsmshd19 -lnro19 `cat /opt/app/oracle/product/19.29.0/dbhome_1/lib/ldflags`    -lncrypt19 -lnsgr19 -lnzjs19 -ln19 -lnl19 -lngsmshd19 -lnnzst19 -lzt19 -lztkg19   -lsnls19 -lnls19  -lcore19 -lsnls19 -lnls19 -lcore19 -lsnls19 -lnls19 -lxml19 -lcore19 -lunls19 -lsnls19 -lnls19 -lcore19 -lnls19 `if /usr/bin/ar tv /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/libknlopt.a | grep "kxmnsd.o" > /dev/null 2>&1 ; then echo " " ; else echo "-lordsdo19 -lserver19"; fi` -L/opt/app/oracle/product/19.29.0/dbhome_1/ctx/lib/ -lctxc19 -lctx19 -lzx19 -lgx19 -lctx19 -lzx19 -lgx19 -lclscest19 -loevm -lclsra19 -ldbcfg19 -lhasgen19 -lskgxn2 -lnnzst19 -lzt19 -lxml19 -lgeneric19 -locr19 -locrb19 -locrutl19 -lhasgen19 -lskgxn2 -lnnzst19 -lzt19 -lxml19 -lgeneric19  -lgeneric19 -lorazip -loraz -llzopro5 -lorabz2 -lorazstd -loralz4 -lipp_z -lipp_bz2 -lippdc -lipps -lippcore  -lippcp -lsnls19 -lnls19  -lcore19 -lsnls19 -lnls19 -lcore19 -lsnls19 -lnls19 -lxml19 -lcore19 -lunls19 -lsnls19 -lnls19 -lcore19 -lnls19 -lsnls19 -lunls19  -lsnls19 -lnls19  -lcore19 -lsnls19 -lnls19 -lcore19 -lsnls19 -lnls19 -lxml19 -lcore19 -lunls19 -lsnls19 -lnls19 -lcore19 -lnls19 -lasmclnt19 -lcommon19 -lcore19  -ledtn19 -laio -lons  -lmql1 -lipc1    -lfthread19    `cat /opt/app/oracle/product/19.29.0/dbhome_1/lib/sysliblist` -Wl,-rpath,/opt/app/oracle/product/19.29.0/dbhome_1/lib -lm    `cat /opt/app/oracle/product/19.29.0/dbhome_1/lib/sysliblist` -ldl -lm   -L/opt/app/oracle/product/19.29.0/dbhome_1/lib `test -x /usr/bin/hugeedit -a -r /usr/lib64/libhugetlbfs.so && test -r /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/shugetlbfs.o && echo -Wl,-zcommon-page-size=2097152 -Wl,-zmax-page-size=2097152 -lhugetlbfs`
rm -f /opt/app/oracle/product/19.29.0/dbhome_1/bin/oracle
mv /opt/app/oracle/product/19.29.0/dbhome_1/rdbms/lib/oracle /opt/app/oracle/product/19.29.0/dbhome_1/bin/oracle
chmod 6751 /opt/app/oracle/product/19.29.0/dbhome_1/bin/oracle
(if [ ! -f /opt/app/oracle/product/19.29.0/dbhome_1/bin/crsd.bin ]; then \
    getcrshome="/opt/app/oracle/product/19.29.0/dbhome_1/srvm/admin/getcrshome" ; \
    if [ -f "$getcrshome" ]; then \
        crshome="`$getcrshome`"; \
        if [ -n "$crshome" ]; then \
            if [ $crshome != /opt/app/oracle/product/19.29.0/dbhome_1 ]; then \
                oracle="/opt/app/oracle/product/19.29.0/dbhome_1/bin/oracle"; \
                $crshome/bin/setasmgidwrap oracle_binary_path=$oracle; \
            fi \
        fi \
    fi \
fi\
);

gcc and gcc-c++ are not required on Oracle Linux and Red Hat Enterprise Linux (refer 2668780.1 and 2982833.1).


This appears to be an issue introduced with 19.29. No such issue with RU 19.28.

echo $ORACLE_HOME
/opt/app/oracle/product/19.28.0/dbhome_1

cd $ORACLE_HOME/rdbms/lib
make -f ins_rdbms.mk uniaud_on ioracle ORACLE_HOME=$ORACLE_HOME
/usr/bin/ar cr /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/libknlopt.a /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/kzaiang.o
chmod 755 /opt/app/oracle/product/19.28.0/dbhome_1/bin
cd /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/;\
/usr/bin/ar r  /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/libknlopt.a `/usr/bin/ar t /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/libknlopt.a` ;

 - Linking Oracle
rm -f /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/oracle
/opt/app/oracle/product/19.28.0/dbhome_1/bin/orald  -o /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/oracle -m64 -z noexecstack -Wl,--disable-new-dtags -L/opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/ -L/opt/app/oracle/product/19.28.0/dbhome_1/lib/ -L/opt/app/oracle/product/19.28.0/dbhome_1/lib/stubs/   -Wl,-E /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/opimai.o /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/ssoraed.o /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/ttcsoi.o -Wl,--whole-archive -lperfsrv19 -Wl,--no-whole-archive /opt/app/oracle/product/19.28.0/dbhome_1/lib/nautab.o /opt/app/oracle/product/19.28.0/dbhome_1/lib/naeet.o /opt/app/oracle/product/19.28.0/dbhome_1/lib/naect.o /opt/app/oracle/product/19.28.0/dbhome_1/lib/naedhs.o /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/config.o  -ldmext -lserver19 -lodm19 -lofs -lcell19 -lnnet19 -lskgxp19 -lsnls19 -lnls19  -lcore19 -lsnls19 -lnls19 -lcore19 -lsnls19 -lnls19 -lxml19 -lcore19 -lunls19 -lsnls19 -lnls19 -lcore19 -lnls19 -lclient19  -lvsnst19 -lcommon19 -lgeneric19 -lknlopt -loraolap19 -lskjcx19 -lslax19 -lpls19  -lrt -lplp19 -ldmext -lserver19 -lclient19  -lvsnst19 -lcommon19 -lgeneric19 `if [ -f /opt/app/oracle/product/19.28.0/dbhome_1/lib/libavserver19.a ] ; then echo "-lavserver19" ; else echo "-lavstub19"; fi` `if [ -f /opt/app/oracle/product/19.28.0/dbhome_1/lib/libavclient19.a ] ; then echo "-lavclient19" ; fi` -lknlopt -lslax19 -lpls19  -lrt -lplp19 -ljavavm19 -lserver19  -lwwg  `cat /opt/app/oracle/product/19.28.0/dbhome_1/lib/ldflags`    -lncrypt19 -lnsgr19 -lnzjs19 -ln19 -lnl19 -lngsmshd19 -lnro19 `cat /opt/app/oracle/product/19.28.0/dbhome_1/lib/ldflags`    -lncrypt19 -lnsgr19 -lnzjs19 -ln19 -lnl19 -lngsmshd19 -lnnzst19 -lzt19 -lztkg19 -lmm -lsnls19 -lnls19  -lcore19 -lsnls19 -lnls19 -lcore19 -lsnls19 -lnls19 -lxml19 -lcore19 -lunls19 -lsnls19 -lnls19 -lcore19 -lnls19 -lztkg19 `cat /opt/app/oracle/product/19.28.0/dbhome_1/lib/ldflags`    -lncrypt19 -lnsgr19 -lnzjs19 -ln19 -lnl19 -lngsmshd19 -lnro19 `cat /opt/app/oracle/product/19.28.0/dbhome_1/lib/ldflags`    -lncrypt19 -lnsgr19 -lnzjs19 -ln19 -lnl19 -lngsmshd19 -lnnzst19 -lzt19 -lztkg19   -lsnls19 -lnls19  -lcore19 -lsnls19 -lnls19 -lcore19 -lsnls19 -lnls19 -lxml19 -lcore19 -lunls19 -lsnls19 -lnls19 -lcore19 -lnls19 `if /usr/bin/ar tv /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/libknlopt.a | grep "kxmnsd.o" > /dev/null 2>&1 ; then echo " " ; else echo "-lordsdo19 -lserver19"; fi` -L/opt/app/oracle/product/19.28.0/dbhome_1/ctx/lib/ -lctxc19 -lctx19 -lzx19 -lgx19 -lctx19 -lzx19 -lgx19 -lclscest19 -loevm -lclsra19 -ldbcfg19 -lhasgen19 -lskgxn2 -lnnzst19 -lzt19 -lxml19 -lgeneric19 -locr19 -locrb19 -locrutl19 -lhasgen19 -lskgxn2 -lnnzst19 -lzt19 -lxml19 -lgeneric19  -lgeneric19 -lorazip -loraz -llzopro5 -lorabz2 -lorazstd -loralz4 -lipp_z -lipp_bz2 -lippdc -lipps -lippcore  -lippcp -lsnls19 -lnls19  -lcore19 -lsnls19 -lnls19 -lcore19 -lsnls19 -lnls19 -lxml19 -lcore19 -lunls19 -lsnls19 -lnls19 -lcore19 -lnls19 -lsnls19 -lunls19  -lsnls19 -lnls19  -lcore19 -lsnls19 -lnls19 -lcore19 -lsnls19 -lnls19 -lxml19 -lcore19 -lunls19 -lsnls19 -lnls19 -lcore19 -lnls19 -lasmclnt19 -lcommon19 -lcore19  -ledtn19 -laio -lons  -lmql1 -lipc1    -lfthread19    `cat /opt/app/oracle/product/19.28.0/dbhome_1/lib/sysliblist` -Wl,-rpath,/opt/app/oracle/product/19.28.0/dbhome_1/lib -lm    `cat /opt/app/oracle/product/19.28.0/dbhome_1/lib/sysliblist` -ldl -lm   -L/opt/app/oracle/product/19.28.0/dbhome_1/lib `test -x /usr/bin/hugeedit -a -r /usr/lib64/libhugetlbfs.so && test -r /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/shugetlbfs.o && echo -Wl,-zcommon-page-size=2097152 -Wl,-zmax-page-size=2097152 -lhugetlbfs`
rm -f /opt/app/oracle/product/19.28.0/dbhome_1/bin/oracle
mv /opt/app/oracle/product/19.28.0/dbhome_1/rdbms/lib/oracle /opt/app/oracle/product/19.28.0/dbhome_1/bin/oracle
chmod 6751 /opt/app/oracle/product/19.28.0/dbhome_1/bin/oracle
(if [ ! -f /opt/app/oracle/product/19.28.0/dbhome_1/bin/crsd.bin ]; then \
    getcrshome="/opt/app/oracle/product/19.28.0/dbhome_1/srvm/admin/getcrshome" ; \
    if [ -f "$getcrshome" ]; then \
        crshome="`$getcrshome`"; \
        if [ -n "$crshome" ]; then \
            if [ $crshome != /opt/app/oracle/product/19.28.0/dbhome_1 ]; then \
                oracle="/opt/app/oracle/product/19.28.0/dbhome_1/bin/oracle"; \
                $crshome/bin/setasmgidwrap oracle_binary_path=$oracle; \
            fi \
        fi \
    fi \
fi\
);

23ai JDBC Driver Does Not Consider RETRY_COUNT and RETRY_DELAY

RETRY_COUNT and RETRY_DELAY are key to setting up JDBC client failover setup. Adjusting these parameters allow connection pool to wait for the duration planned outtage without issuing an error message.
However, 23ai JDBC driver has change in behaviour that by default it ignores these two parameters (3073421.1). Instead connection wait time needs to be set at the UCP using the

setConnectionWaitTimeout

method. So if moving into 23ai driver set Connection Wait Timeout equal to or slightly higher than the total outtage time (which use to be RETRY_COUNT x RETRY_DELAY).
Above may require code changes depending on how UCP pool is used in the application. If old behaviour of using RETRY_COUNT and RETRY_DELAY is perfered then it could be enabled on 23ai JDBC driver by setting the following JVM parameter

-Doracle.ucp.createConnectionInBorrowThread=true

Useful metalink notes
Universal Connection Pool Times Out Prematurely During A Data Guard Switchover Test [ID 3073421.1]

Related Post
JDBC Client Failover in Data Guard Configuration with PDBs

Restore Standby Database from Standby Backups

This post shows the steps for full restoring (controlfile + data files) for a standby database using standby database backups. The same could be achieved using restore from service (2283978.1). However, this method is useful when the database size is large and high network latencies are invovled.

1. As the first step disable log apply and transport.

DGMGRL> edit database fsfodr set state='apply-off';
Succeeded.
DGMGRL> edit database fsfopr set state='transport-off';
Succeeded.
DGMGRL>

2. Start the standby database in nomount mode and restore the standby controlfile.

$ rman target /
RMAN > startup nomount

RMAN> restore standby controlfile from '/opt/backup/fsfodr/full_c-1245564449-20230913-02.ctl';

Starting restore at 13-SEP-23
using target database control file instead of recovery catalog
allocated channel: ORA_DISK_1
channel ORA_DISK_1: SID=2836 device type=DISK

channel ORA_DISK_1: restoring control file
channel ORA_DISK_1: restore complete, elapsed time: 00:00:02
output file name=+DATA/FSFODR/CONTROLFILE/current.341.1147435431
output file name=+FRA/FSFODR/CONTROLFILE/current.573.1147435431
Finished restore at 13-SEP-23

3. Mount the database.

RMAN> alter database mount;

4. Since the standby controfile was restored from a backup taken on standby database no need to catalog backup file location. The controlfile is aware of the backup locations. Run a restore and recover statements.

RMAN> run {
2> restore database;
3> recover database;
4> }

5. Clear the online logfiles on the standby database.

SQL> begin
for log_cur in ( select group# group_no from v$log )
loop
execute immediate 'alter database clear logfile group '||log_cur.group_no;
end loop;
end;
/

PL/SQL procedure successfully completed.

SQL> select * from v$log;

    GROUP#    THREAD#  SEQUENCE#      BYTES  BLOCKSIZE    MEMBERS ARC STATUS           FIRST_CHANGE# FIRST_TIM NEXT_CHANGE# NEXT_TIME     CON_ID
---------- ---------- ---------- ---------- ---------- ---------- --- ---------------- ------------- --------- ------------ --------- ----------
         1          1          0  104857600        512          2 YES UNUSED                 2635375 13-SEP-23      2635378 13-SEP-23          0
         2          1          0  104857600        512          2 YES UNUSED                 2635378 13-SEP-23      2636371 13-SEP-23          0
         5          1          0  104857600        512          2 YES UNUSED                 2636825 13-SEP-23   9.2954E+18                    0
         4          1          0  104857600        512          2 YES UNUSED                 2636470 13-SEP-23      2636825 13-SEP-23          0
         3          1          0  104857600        512          2 YES UNUSED                 2636371 13-SEP-23      2636470 13-SEP-23          0

6. Clear the standby logfiles

begin
for log_cur in ( select group# group_no from v$standby_log )
loop
execute immediate 'alter database clear logfile group '||log_cur.group_no;
end loop;
end;
/


PL/SQL procedure successfully completed.

SQL> select * from v$standby_log;

    GROUP# DBID                                        THREAD#  SEQUENCE#      BYTES  BLOCKSIZE       USED ARC STATUS     FIRST_CHANGE# FIRST_TIM NEXT_CHANGE# NEXT_TIME LAST_CHANGE# LAST_TIME     CON_ID
---------- ---------------------------------------- ---------- ---------- ---------- ---------- ---------- --- ---------- ------------- --------- ------------ --------- ------------ --------- ----------
         6 UNASSIGNED                                        1          0  104857600        512          0 NO  UNASSIGNED                                                                                0
         7 UNASSIGNED                                        1          0  104857600        512          0 YES UNASSIGNED                                                                                0
         8 UNASSIGNED                                        1          0  104857600        512          0 YES UNASSIGNED                                                                                0
         9 UNASSIGNED                                        1          0  104857600        512          0 YES UNASSIGNED                                                                                0
        10 UNASSIGNED                                        1          0  104857600        512          0 YES UNASSIGNED                                                                                0
        11 UNASSIGNED                                        1          0  104857600        512          0 YES UNASSIGNED                                                                                0

6 rows selected.

7. Enable log transport and redo apply

DGMGRL>  edit database fsfopr set state='transport-on';
Succeeded.
DGMGRL>  edit database fsfodr set state='apply-on';
Succeeded.

8. Check data guard configuration status and valdiate the standby database

DGMGRL> show configuration

Configuration - fsfo_dg

  Protection Mode: MaxAvailability
  Members:
  fsfopr - Primary database
    fsfodr - Physical standby database

Fast-Start Failover:  Disabled

Configuration Status:
SUCCESS   (status updated 25 seconds ago)

DGMGRL> validate database fsfodr;

  Database Role:     Physical standby database
  Primary Database:  fsfopr

  Ready for Switchover:  Yes
  Ready for Failover:    Yes (Primary Running)

  Managed by Clusterware:
    fsfopr:  YES
    fsfodr:  YES

Useful metalink notes
Creating a Physical Standby database using RMAN restore database from service [ID 2283978.1]