In [ ]:
# %%bash
# cp $MYDB/ND_Dados_DBs/INOSS/T_*xlsx 
# mkdir -p CSV
# mkdir -p Reports
# python  xlsx2csv.py
In [2]:
%matplotlib  inline
import seaborn as sns
import matplotlib.pyplot as plt
from datetime import datetime
import pandas as pd
import matplotlib

df = pd.read_excel("CSV/T_LOG1.csv", sep=",")
In [3]:
df  
Out[3]:
FN_ID FD_DATE FN_SEVERITY FT_APP FT_DESCRIPTION FB_SPECIFICINFO FN_LOGONLY FT_EVENTTIMESTAMP FT_APPID FD_DAY FD_MONTH FD_YEAR FD_WEEKDAY FD_DAYN
0 13970121 2020-04-21 4 SIP CLEAR: PSLE0052-PISAAMI (BLOB) 0 2274645636357819887 NaN 21 4 20 1 21
1 13970122 2020-04-21 3 SIP CLEAR: PSLB0052-PISAAMA (BLOB) 0 2274645636357819888 NaN 21 4 20 1 21
2 13970123 2020-04-21 4 SIP CLEAR: PSLB0052-PISAAMI (BLOB) 0 2274645636357819889 NaN 21 4 20 1 21
3 13970124 2020-04-21 4 SIP CLEAR: PFAN0062-PISAAMI (BLOB) 0 2274645636357819890 NaN 21 4 20 1 21
4 13970125 2020-04-21 3 SIP CLEAR: PFAN0062-PISAAMA (BLOB) 0 2274645636374597107 NaN 21 4 20 1 21
... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
37659 13974094 2020-04-30 9 INTERCOM Danmarks plass, DAP1: Active Call (BLOB) 0 2274666815395219634 NaN 30 4 20 3 30
37660 13974095 2020-04-30 9 INTERCOM OCC Intercom CR, sipddms: Clear Call (BLOB) 0 2274666815529437364 NaN 30 4 20 3 30
37661 13974096 2020-04-30 9 INTERCOM Danmarks plass, DAP1: Clear Call (BLOB) 0 2274666815529437365 NaN 30 4 20 3 30
37662 13974097 2020-04-30 4 STV OCC CCTV, EHP: IO inactive (BLOB) 0 2274666815562991798 0.0 30 4 20 3 30
37663 13974098 2020-04-30 0 STV Monitor group changed (BLOB) 0 2274666815562991799 0.0 30 4 20 3 30

37664 rows × 14 columns

In [4]:
df.describe()
Out[4]:
FN_ID FN_SEVERITY FN_LOGONLY FT_EVENTTIMESTAMP FT_APPID FD_DAY FD_MONTH FD_YEAR FD_WEEKDAY FD_DAYN
count 3.766400e+04 37664.000000 37664.0 3.766400e+04 1.964600e+04 37664.000000 37664.000000 37664.0 37664.000000 37664.00000
mean 1.396959e+07 3.885753 0.0 2.265282e+18 3.133500e+07 13.892523 4.176083 20.0 3.055119 12.78733
std 1.087309e+04 2.468931 0.0 1.456216e+17 4.705947e+07 6.841038 0.744043 0.0 1.891373 7.62205
min 1.395076e+07 0.000000 0.0 1.583845e+12 -1.000000e+00 1.000000 3.000000 20.0 0.000000 3.00000
25% 1.396018e+07 3.000000 0.0 2.274604e+18 0.000000e+00 9.000000 4.000000 20.0 1.000000 5.00000
50% 1.396959e+07 4.000000 0.0 2.274644e+18 0.000000e+00 13.000000 4.000000 20.0 3.000000 13.00000
75% 1.397901e+07 4.000000 0.0 2.274692e+18 1.020044e+08 19.000000 5.000000 20.0 5.000000 19.00000
max 1.398842e+07 9.000000 0.0 2.274725e+18 1.020093e+08 30.000000 5.000000 20.0 6.000000 30.00000
In [5]:
ax = sns.countplot(df['FN_SEVERITY'])
In [6]:
#Apps
ax = sns.countplot(df['FT_APP'])
In [7]:
#day
ax = sns.countplot(df['FD_DAY'])
In [8]:
#week day
ax = sns.countplot(df['FD_WEEKDAY'])
In [9]:
def plotAPP(df1):
    df1['Y'] = df1.loc[:,'FT_APP'].astype('category')
    vs = df1.Y.cat.categories
   
    l = len(vs)
    for ev in vs:
        dat = df1[df1['Y']==ev]
        ax = sns.countplot(dat.loc[:,'FN_SEVERITY'])
        ax.set_title('SEVERITY IN ' + ev) 
        plt.show()
       

def plotWeek(df1):
    df1['Y'] = df1.loc[:,'FD_WEEKDAY'].astype('category')
    vs = df1.Y.cat.categories
    fig, axes =plt.subplots(7,1,figsize=(10,30))
    print(vs)
    weekday =0
    for ev in vs:
        dat = df1[df1['Y']==ev]
        sns.countplot(dat.loc[:,'FN_SEVERITY'],ax=axes[weekday],)
       
        axes[weekday].set_title('SEVERITY IN WEEK DAY ' + str(ev)) 
        weekday+=1
        
    plt.show()
       

       
def plotMonth(df1):
    df1['Y'] = df1.loc[:,'FD_MONTH'].astype('category')
    vs = df1.Y.cat.categories
   
    l = len(vs)
    for ev in vs:
        dat = df1[df1['Y']==ev]
        ax = sns.countplot(dat.loc[:,'FN_SEVERITY'])
        ax.set_title('SEVERITY IN Month ' + str(ev)) 
        plt.show()


def plotDay(df1):
    df1['Y'] = df1.loc[:,'FN_SEVERITY'].astype('category')
    vs = df1.Y.cat.categories
   
    l = len(vs)
    for ev in vs:
        dat = df1[df1['Y']==ev]
        ax = sns.countplot(dat.loc[:,'FD_DAYN'])
        ax.set_title('SEVERITY '+ str(ev)+ 'since logs started') 
        plt.show()
In [10]:
plotAPP(df)
In [11]:
plotWeek(df)
Int64Index([0, 1, 2, 3, 4, 5, 6], dtype='int64')
In [12]:
plotMonth(df)
In [13]:
plotDay(df)
In [14]:
#analise Proprieties
df1 = pd.read_csv("CSV/T_LOG1_PROPERTIES.csv", sep=",")
In [15]:
df1
Out[15]:
FN_ID FN_LOG_ID FT_KEY FT_VAL
0 113449683 13969836 INOSSEvent TRUE
1 113449684 13969836 EventAppId 102006630
2 113449685 13969836 EventID 11
3 113449686 13969836 EventSeverity 3
4 113449687 13969836 EventStatus PRESENT
... ... ... ... ...
537294 113511807 13973716 STVStationID T11
537295 113511808 13973716 STVEquipID TSAT018WCCTVC1
537296 113511809 13973716 EventDescription_en_GB Såtamyri underpass, Camera CAM Track1: communi...
537297 113511810 13973716 EventDescription_nb_NO Såtamyri underpass, Camera CAM Track1:kommunik...
537298 113511811 13973716 EventDate 2020-04-29 18:03:20

537299 rows × 4 columns

In [16]:
df1['Y'] = df1.loc[:,'FT_KEY'].astype('category')
df1.Y.describe()
df1.Y.value_counts()
Out[16]:
EventSeverity             37659
EventDate                 37654
EventStatus               37569
EventDescription          37521
EventApp                  36193
EventDescription_en_GB    24673
EventDescription_nb_NO    23019
EventAppId                19643
EventOrigID               19494
EventEqID                 19491
EventID                   19472
BISProcessed              16577
PINTNoShow                13282
STVObjID                   9469
STVEventParam              9469
STVAlarmType               9468
STVStationID               9467
STVEquipID                 9467
EventAppModule             9422
Auditing                   9407
EventAlarmClear            9338
EventOrig                  9338
EventSystem                9338
INOSSEvent                 9338
EventEqKind                9337
EventNotify                9337
EventSubtype               9336
EventRefresh               9335
EventIDs                   9334
STVServerOnly              7294
EventNumDesc               6596
STVScadaTag                6418
EventSrcDate               6392
StationId                  2017
EquipId                    2017
groupId                    1808
AuditActionParam           1659
AuditAction                1659
Username                   1659
WSID                       1659
AuditAction_en_GB          1658
EventTypeId                1188
STVServerAlarmStatus        876
AuditAction_nb_NO           563
AuditEquipment              486
AlarmApp                    308
EquipID                     155
AlarmType                   155
EventAppType                153
EventZones                   71
EventParam                   16
ObjID                        14
StationID                    14
sourceId                      8
targetId                      8
AuditTargetLocation           1
Name: Y, dtype: int64
In [17]:
df1['Y'] = df1.loc[:,'FT_VAL'].astype('category')
df1.Y.describe()
df1.Y.value_counts()
Out[17]:
0                      54287
1                      22542
CLEAR                  19217
4                      19139
FALSE                  18101
                       ...  
2020-05-24 01:32:45        1
2020-05-24 01:46:21        1
2020-05-24 01:46:23        1
2020-04-14 14:58:47        1
wer2                       1
Name: Y, Length: 28175, dtype: int64
In [18]:
df.rename(columns={'FN_ID': 'FN_LOG_ID'}, inplace=True)
dfmerged = pd.merge(df, df1, on=['FN_LOG_ID'],how="right")
In [19]:
dfmerged
Out[19]:
FN_LOG_ID FD_DATE FN_SEVERITY FT_APP FT_DESCRIPTION FB_SPECIFICINFO FN_LOGONLY FT_EVENTTIMESTAMP FT_APPID FD_DAY FD_MONTH FD_YEAR FD_WEEKDAY FD_DAYN Y_x FN_ID FT_KEY FT_VAL Y_y
0 13970121 2020-04-21 4 SIP CLEAR: PSLE0052-PISAAMI (BLOB) 0 2274645636357819887 NaN 21 4 20 1 21 4 113455184 EventID 235 235
1 13970121 2020-04-21 4 SIP CLEAR: PSLE0052-PISAAMI (BLOB) 0 2274645636357819887 NaN 21 4 20 1 21 4 113455185 EventSeverity 4 4
2 13970121 2020-04-21 4 SIP CLEAR: PSLE0052-PISAAMI (BLOB) 0 2274645636357819887 NaN 21 4 20 1 21 4 113455186 EventStatus CLEAR CLEAR
3 13970121 2020-04-21 4 SIP CLEAR: PSLE0052-PISAAMI (BLOB) 0 2274645636357819887 NaN 21 4 20 1 21 4 113455187 BISProcessed NaN NaN
4 13970121 2020-04-21 4 SIP CLEAR: PSLE0052-PISAAMI (BLOB) 0 2274645636357819887 NaN 21 4 20 1 21 4 113455188 EventOrigID 1010000 1010000
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
537294 13974098 2020-04-30 0 STV Monitor group changed (BLOB) 0 2274666815562991799 0.0 30 4 20 3 30 0 113515791 groupId -1 -1
537295 13974098 2020-04-30 0 STV Monitor group changed (BLOB) 0 2274666815562991799 0.0 30 4 20 3 30 0 113515792 PINTNoShow TRUE TRUE
537296 13974098 2020-04-30 0 STV Monitor group changed (BLOB) 0 2274666815562991799 0.0 30 4 20 3 30 0 113515793 EventDescription_en_GB Monitor group changed Monitor group changed
537297 13974098 2020-04-30 0 STV Monitor group changed (BLOB) 0 2274666815562991799 0.0 30 4 20 3 30 0 113515794 EventDescription_nb_NO Skjermgruppe endret Skjermgruppe endret
537298 13974098 2020-04-30 0 STV Monitor group changed (BLOB) 0 2274666815562991799 0.0 30 4 20 3 30 0 113515795 EventDate 2020-04-30 22:43:04 2020-04-30 22:43:04

537299 rows × 19 columns

In [39]:
n_df=pd.Index(df.FN_LOG_ID)
for i in df1:
    #iprint(i)
    df[df.FN_LOG_ID==i[0,"FN_ID"]][i.FT_KEY]=df1.FN_ID[0,"FT_VAL"]    
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-39-b4e38ed19eda> in <module>
      2 for i in df1:
      3     #iprint(i)
----> 4     df[df.FN_LOG_ID==i[0,"FN_ID"]][i.FT_KEY]=df1.FN_ID[0,"FT_VAL"]

/usr/lib64/python3.8/site-packages/pandas/core/series.py in __getitem__(self, key)
   1069         key = com.apply_if_callable(key, self)
   1070         try:
-> 1071             result = self.index.get_value(self, key)
   1072 
   1073             if not is_scalar(result):

/usr/lib64/python3.8/site-packages/pandas/core/indexes/base.py in get_value(self, series, key)
   4728         k = self._convert_scalar_indexer(k, kind="getitem")
   4729         try:
-> 4730             return self._engine.get_value(s, k, tz=getattr(series.dtype, "tz", None))
   4731         except KeyError as e1:
   4732             if len(self) > 0 and (self.holds_integer() or self.is_boolean()):

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_value()

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_value()

pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

pandas/_libs/index_class_helper.pxi in pandas._libs.index.Int64Engine._check_type()

KeyError: (0, 'FT_VAL')
In [25]:
df1
Out[25]:
FN_ID FN_LOG_ID FT_KEY FT_VAL Y
0 113449683 13969836 INOSSEvent TRUE TRUE
1 113449684 13969836 EventAppId 102006630 102006630
2 113449685 13969836 EventID 11 11
3 113449686 13969836 EventSeverity 3 3
4 113449687 13969836 EventStatus PRESENT PRESENT
... ... ... ... ... ...
537294 113511807 13973716 STVStationID T11 T11
537295 113511808 13973716 STVEquipID TSAT018WCCTVC1 TSAT018WCCTVC1
537296 113511809 13973716 EventDescription_en_GB Såtamyri underpass, Camera CAM Track1: communi... Såtamyri underpass, Camera CAM Track1: communi...
537297 113511810 13973716 EventDescription_nb_NO Såtamyri underpass, Camera CAM Track1:kommunik... Såtamyri underpass, Camera CAM Track1:kommunik...
537298 113511811 13973716 EventDate 2020-04-29 18:03:20 2020-04-29 18:03:20

537299 rows × 5 columns

In [ ]: