nwickelmaier/mtt_haum
nwickelmaier
/
mtt_haum
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Mostly updated file paths

This commit is contained in:
Nora Wickelmaier 2024-03-22 12:07:45 +01:00
parent b762968774
commit 43c7f34645
7 changed files with 89 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
code/01_preprocessing.R
View File

@ -8,8 +8,8 @@
# ../data/metadata/feiertage.csv # ../data/metadata/feiertage.csv
# ../data/metadata/schulferien_2016-2018_NI.csv # ../data/metadata/schulferien_2016-2018_NI.csv
# ../data/metadata/schulferien_2019-2025_NI.csv # ../data/metadata/schulferien_2019-2025_NI.csv
# output: raw_logfiles_<timestamp>.csv # output: results/raw_logfiles_<timestamp>.csv
# event_logfiles_<timestamp>.csv # results/event_logfiles_<timestamp>.csv
# #
# last mod: 2024-02-23, NW # last mod: 2024-02-23, NW
@ -29,12 +29,12 @@ folders <- dir(path)
datraw <- parse_logfiles(folders, path) datraw <- parse_logfiles(folders, path)
# 91 corrupt lines have been found and removed from the data set # 91 corrupt lines have been found and removed from the data set
# datraw <- read.table("results/haum/raw_logfiles_2023-10-25_16-20-45.csv", # datraw <- read.table("results/raw_logfiles_2023-10-25_16-20-45.csv",
# sep = ";", header = TRUE) # sep = ";", header = TRUE)
## Export data ## Export data
write.table(datraw, paste0("results/haum/raw_logfiles_", now, ".csv"), write.table(datraw, paste0("results/raw_logfiles_", now, ".csv"),
sep = ";", row.names = FALSE) sep = ";", row.names = FALSE)
#--------------- (2) Create event logs --------------- #--------------- (2) Create event logs ---------------
@ -131,6 +131,6 @@ dat2 <- dat2[order(dat2$fileId.start, dat2$date.start, dat2$timeMs.start), ]
## Export data ## Export data
write.table(dat2, paste0("results/haum/event_logfiles_", now, ".csv"), write.table(dat2, paste0("results/event_logfiles_", now, ".csv"),
sep = ";", row.names = FALSE) sep = ";", row.names = FALSE)

8
code/02_descriptives.R
View File

@ -9,8 +9,8 @@
# (3.4) Artwork sequences # (3.4) Artwork sequences
# (3.5) Topics # (3.5) Topics
# #
# input: results/haum/event_logfiles_2024-02-21_16-07-33.csv # input: results/event_logfiles_2024-02-21_16-07-33.csv
# results/haum/raw_logfiles_2024-02-21_16-07-33.csv # results/raw_logfiles_2024-02-21_16-07-33.csv
# output: results/figures/counts_item.pdf # output: results/figures/counts_item.pdf
# results/figures/counts_item_firsttouch.pdf # results/figures/counts_item_firsttouch.pdf
# results/figures/duration.pdf # results/figures/duration.pdf
@ -41,7 +41,7 @@
#--------------- (1) Read data --------------- #--------------- (1) Read data ---------------
datlogs <- read.table("results/haum/event_logfiles_2024-02-21_16-07-33.csv", datlogs <- read.table("results/event_logfiles_2024-02-21_16-07-33.csv",
colClasses = c("character", "character", "POSIXct", colClasses = c("character", "character", "POSIXct",
"POSIXct", "character", "integer", "POSIXct", "character", "integer",
"numeric", "character", "character", "numeric", "character", "character",
@ -54,7 +54,7 @@ datlogs$event <- factor(datlogs$event, levels = c("move", "flipCard",
"openTopic", "openTopic",
"openPopup")) "openPopup"))
datraw <- read.table("results/haum/raw_logfiles_2024-02-21_16-07-33.csv", datraw <- read.table("results/raw_logfiles_2024-02-21_16-07-33.csv",
sep = ";", header = TRUE) sep = ";", header = TRUE)
# Add weekdays to data frame # Add weekdays to data frame

76
code/04_conformance-checking.py
View File

@ -1,25 +1,24 @@
# 04_conformance-checking.py # 04_conformance-checking.py
# #
# content: (1) Load data and create event log # content: (1) Load data and create event log
# (2) Infos for items # (2) Check against normative Petri Net
# #
# input: results/haum/event_logfiles_2024-02-21_16-07-33.csv # input: results/event_logfiles_2024-02-21_16-07-33.csv
# results/haum/conformative_petrinet_con.pnml # results/normative_petrinet.pnml
# output: results/processmaps/dfg_complete_python.png # output: results/eval_all-miners_complete.csv
# results/eval_all-miners_complete.csv
# results/eval_all-miners_clean.csv # results/eval_all-miners_clean.csv
# results/processmaps/petrinet_conformative.png # ../../thesis/figures/petrinet_normative.png
# results/processmaps/petrinet_heuristics_clean.png # ../../thesis/figures/petrinet_heuristics_clean.png
# results/processmaps/petrinet_alpha_clean.png # ../../thesis/figures/petrinet_alpha_clean.png
# results/processmaps/petrinet_inductive_clean.png # ../../thesis/figures/petrinet_inductive_clean.png
# results/processmaps/petrinet_ilp_clean.png # ../../thesis/figures/petrinet_ilp_clean.png
# results/processmaps/bpmn_conformative.png # ../../thesis/figures/bpmn_normative.png
# results/processmaps/bpmn_inductive_clean.png # ../../thesis/figures/bpmn_inductive_clean.png
# results/processmaps/bpmn_ilp_clean.png # ../../thesis/figures/bpmn_ilp_clean.png
# results/processmaps/bpmn_alpha_clean.png # ../../thesis/figures/bpmn_alpha_clean.png
# results/processmaps/bpmn_heuristics_clean.png # ../../thesis/figures/bpmn_heuristics_clean.png
# #
# last mod: 2024-03-06 # last mod: 2024-03-22
import pm4py import pm4py
import pandas as pd import pandas as pd
@ -29,13 +28,13 @@ from python_helpers import eval_pm, pn_infos_miner
#--------------- (1) Load data and create event logs --------------- #--------------- (1) Load data and create event logs ---------------
dat = pd.read_csv("results/haum/event_logfiles_2024-02-21_16-07-33.csv", sep = ";") dat = pd.read_csv("results/event_logfiles_2024-02-21_16-07-33.csv", sep = ";")
event_log = pm4py.format_dataframe(dat, case_id = "path", event_log = pm4py.format_dataframe(dat, case_id = "path",
activity_key = "event", activity_key = "event",
timestamp_key = "date.start") timestamp_key = "date.start")
###### Descriptives of log data ###### ## Descriptives of log data
# Distribution of events # Distribution of events
event_log.event.value_counts() event_log.event.value_counts()
@ -57,9 +56,9 @@ len(variants_no_move)
sorted_variants_no_move = dict(sorted(variants_no_move.items(), key=lambda item: item[1], reverse = True)) sorted_variants_no_move = dict(sorted(variants_no_move.items(), key=lambda item: item[1], reverse = True))
{k: sorted_variants_no_move[k] for k in list(sorted_variants_no_move)[:20]} {k: sorted_variants_no_move[k] for k in list(sorted_variants_no_move)[:20]}
###### Check against "conformative" Petri Net ###### #--------------- (2) Check against normative Petri Net ---------------
basenet, initial_marking, final_marking = pm4py.read_pnml("results/haum/conformative_petrinet_con.pnml") basenet, initial_marking, final_marking = pm4py.read_pnml("results/normative_petrinet.pnml")
# TBR # TBR
replayed_traces = pm4py.conformance_diagnostics_token_based_replay(event_log, basenet, initial_marking, final_marking) replayed_traces = pm4py.conformance_diagnostics_token_based_replay(event_log, basenet, initial_marking, final_marking)
@ -93,23 +92,13 @@ event_log[event_log["@@case_index"] == index_broken[0]].item.unique().tolist()
event_log[event_log["@@case_index"] == index_broken[0]]["fileId.start"].unique().tolist() event_log[event_log["@@case_index"] == index_broken[0]]["fileId.start"].unique().tolist()
# --> logging error in raw file # --> logging error in raw file
## Footprints
from pm4py.algo.discovery.footprints import algorithm as footprints_discovery
from pm4py.visualization.footprints import visualizer as fp_visualizer
fp_log = footprints_discovery.apply(event_log, variant=footprints_discovery.Variants.ENTIRE_EVENT_LOG)
fp_net = footprints_discovery.apply(basenet, initial_marking, final_marking)
gviz = fp_visualizer.apply(fp_net, parameters={fp_visualizer.Variants.SINGLE.value.Parameters.FORMAT: "svg"})
fp_visualizer.view(gviz)
efg_graph = pm4py.discover_eventually_follows_graph(event_log)
## Fitting different miners ## Fitting different miners
eval = pd.DataFrame(columns = ["fitness", "precision", "generalizability", eval = pd.DataFrame(columns = ["fitness", "precision", "generalizability",
"simplicity", "sound", "narcs", "ntrans", "simplicity", "sound", "narcs", "ntrans",
"nplaces", "nvariants", "mostfreq"]) "nplaces", "nvariants", "mostfreq"])
for miner in ["conformative", "alpha", "heuristics", "inductive", "ilp"]: for miner in ["normative", "alpha", "heuristics", "inductive", "ilp"]:
eval = pd.concat([eval, pn_infos_miner(event_log, miner)]) eval = pd.concat([eval, pn_infos_miner(event_log, miner)])
eval.to_csv("results/eval_all-miners_complete.csv", sep = ";") eval.to_csv("results/eval_all-miners_complete.csv", sep = ";")
@ -121,7 +110,7 @@ eval_clean = pd.DataFrame(columns = ["fitness", "precision", "generalizability",
"simplicity", "sound", "narcs", "ntrans", "simplicity", "sound", "narcs", "ntrans",
"nplaces", "nvariants", "mostfreq"]) "nplaces", "nvariants", "mostfreq"])
for miner in ["conformative", "alpha", "heuristics", "inductive", "ilp"]: for miner in ["normative", "alpha", "heuristics", "inductive", "ilp"]:
eval_clean = pd.concat([eval_clean, pn_infos_miner(event_log_clean, miner)]) eval_clean = pd.concat([eval_clean, pn_infos_miner(event_log_clean, miner)])
eval_clean.to_csv("results/eval_all-miners_clean.csv", sep = ";") eval_clean.to_csv("results/eval_all-miners_clean.csv", sep = ";")
@ -129,28 +118,27 @@ eval_clean.to_csv("results/eval_all-miners_clean.csv", sep = ";")
## Directly-follows graph ## Directly-follows graph
dfg, start_activities, end_activities = pm4py.discover_dfg(event_log_clean) dfg, start_activities, end_activities = pm4py.discover_dfg(event_log_clean)
pm4py.view_dfg(dfg, start_activities, end_activities) pm4py.view_dfg(dfg, start_activities, end_activities)
pm4py.save_vis_dfg(dfg, start_activities, end_activities, "results/processmaps/dfg_complete_python.png")
## Export petri nets ## Export petri nets
pm4py.vis.save_vis_petri_net(basenet, initial_marking, final_marking, "results/processmaps/petrinet_conformative.png") pm4py.vis.save_vis_petri_net(basenet, initial_marking, final_marking,
"../../thesis/figures/petrinet_normative.png")
h_net, h_im, h_fm = pm4py.discover_petri_net_heuristics(event_log_clean) h_net, h_im, h_fm = pm4py.discover_petri_net_heuristics(event_log_clean)
pm4py.vis.save_vis_petri_net(h_net, h_im, h_fm, "results/processmaps/petrinet_heuristics_clean.png") pm4py.vis.save_vis_petri_net(h_net, h_im, h_fm, "../../thesis/figures/petrinet_heuristics_clean.png")
a_net, a_im, a_fm = pm4py.discover_petri_net_alpha(event_log_clean) a_net, a_im, a_fm = pm4py.discover_petri_net_alpha(event_log_clean)
pm4py.vis.save_vis_petri_net(a_net, a_im, a_fm, "results/processmaps/petrinet_alpha_clean.png") pm4py.vis.save_vis_petri_net(a_net, a_im, a_fm, "../../thesis/figures/petrinet_alpha_clean.png")
i_net, i_im, i_fm = pm4py.discover_petri_net_inductive(event_log_clean) i_net, i_im, i_fm = pm4py.discover_petri_net_inductive(event_log_clean)
pm4py.vis.save_vis_petri_net(i_net, i_im, i_fm, "results/processmaps/petrinet_inductive_clean.png") pm4py.vis.save_vis_petri_net(i_net, i_im, i_fm, "../../thesis/figures/petrinet_inductive_clean.png")
ilp_net, ilp_im, ilp_fm = pm4py.discover_petri_net_ilp(event_log_clean) ilp_net, ilp_im, ilp_fm = pm4py.discover_petri_net_ilp(event_log_clean)
pm4py.vis.save_vis_petri_net(ilp_net, ilp_im, ilp_fm, "results/processmaps/petrinet_ilp_clean.png") pm4py.vis.save_vis_petri_net(ilp_net, ilp_im, ilp_fm, "../../thesis/figures/petrinet_ilp_clean.png")
# convert to BPMN # convert to BPMN
base_bpmn = pm4py.convert.convert_to_bpmn(basenet, initial_marking, final_marking) base_bpmn = pm4py.convert.convert_to_bpmn(basenet, initial_marking, final_marking)
pm4py.vis.save_vis_bpmn(base_bpmn, "results/processmaps/bpmn_conformative.png") pm4py.vis.save_vis_bpmn(base_bpmn, "../../thesis/figures/bpmn_normative.png")
i_bpmn = pm4py.convert.convert_to_bpmn(i_net, i_im, i_fm) i_bpmn = pm4py.convert.convert_to_bpmn(i_net, i_im, i_fm)
pm4py.vis.save_vis_bpmn(i_bpmn, "results/processmaps/bpmn_inductive_clean.png") pm4py.vis.save_vis_bpmn(i_bpmn, "../../thesis/figures/bpmn_inductive_clean.png")
ilp_bpmn = pm4py.convert.convert_to_bpmn(ilp_net, ilp_im, ilp_fm) ilp_bpmn = pm4py.convert.convert_to_bpmn(ilp_net, ilp_im, ilp_fm)
pm4py.vis.save_vis_bpmn(ilp_bpmn, "results/processmaps/bpmn_ilp_clean.png") pm4py.vis.save_vis_bpmn(ilp_bpmn, "../../thesis/figures/bpmn_ilp_clean.png")
a_bpmn = pm4py.convert.convert_to_bpmn(a_net, a_im, a_fm) a_bpmn = pm4py.convert.convert_to_bpmn(a_net, a_im, a_fm)
pm4py.vis.save_vis_bpmn(a_bpmn, "results/processmaps/bpmn_alpha_clean.png") pm4py.vis.save_vis_bpmn(a_bpmn, "../../thesis/figures/bpmn_alpha_clean.png")
h_bpmn = pm4py.convert.convert_to_bpmn(h_net, h_im, h_fm) h_bpmn = pm4py.convert.convert_to_bpmn(h_net, h_im, h_fm)
pm4py.vis.save_vis_bpmn(h_bpmn, "results/processmaps/bpmn_heuristics_clean.png") pm4py.vis.save_vis_bpmn(h_bpmn, "../../thesis/figures/bpmn_heuristics_clean.png")

23
code/05_check-traces.R
View File

@ -5,22 +5,23 @@
# (3) DFG for complete data # (3) DFG for complete data
# (4) Export data frame for analyses # (4) Export data frame for analyses
# #
# input: results/haum/event_logfiles_2024-02-21_16-07-33.csv # input: results/event_logfiles_2024-02-21_16-07-33.csv
# results/haum/raw_logfiles_2024-02-21_16-07-33.csv # results/raw_logfiles_2024-02-21_16-07-33.csv
# output: results/haum/eventlogs_pre-corona_cleaned.RData # output: results/eventlogs_pre-corona_cleaned.RData
# results/haum/eventlogs_pre-corona_cleaned.csv # results/eventlogs_pre-corona_cleaned.csv
# ../../thesis/figures/dfg_complete_WFnet_R.pdf
# #
# last mod: 2024-03-06 # last mod: 2024-03-23
# setwd("C:/Users/nwickelmaier/Nextcloud/Documents/MDS/2023ss/60100_master_thesis/analysis/code") # setwd("C:/Users/nwickelmaier/Nextcloud/Documents/MDS/2023ss/60100_master_thesis/analysis/code")
#--------------- (1) Look at broken trace --------------- #--------------- (1) Look at broken trace ---------------
datraw <- read.table("results/haum/raw_logfiles_2024-02-21_16-07-33.csv", datraw <- read.table("results/raw_logfiles_2024-02-21_16-07-33.csv",
header = TRUE, sep = ";") header = TRUE, sep = ";")
datlogs <- read.table("results/haum/event_logfiles_2024-02-21_16-07-33.csv", datlogs <- read.table("results/event_logfiles_2024-02-21_16-07-33.csv",
colClasses = c("character", "character", "POSIXct", colClasses = c("character", "character", "POSIXct",
"POSIXct", "character", "integer", "POSIXct", "character", "integer",
"numeric", "character", "character", "numeric", "character", "character",
@ -84,7 +85,7 @@ dfg <- processmapR::process_map(alog,
render = FALSE) render = FALSE)
processmapR::export_map(dfg, processmapR::export_map(dfg,
file_name = paste0("results/processmaps/dfg_complete_WFnet_R.pdf"), file_name = paste0("../../thesis/figures/dfg_complete_WFnet_R.pdf"),
file_type = "pdf") file_type = "pdf")
rm(tmp) rm(tmp)
@ -109,10 +110,10 @@ dat <- datlogs[as.Date(datlogs$date.start) < "2020-03-13", ]
# Remove corrupt trace # Remove corrupt trace
dat <- dat[dat$path != 106098, ] dat <- dat[dat$path != 106098, ]
save(dat, file = "results/haum/eventlogs_pre-corona_cleaned.RData") save(dat, file = "results/eventlogs_pre-corona_cleaned.RData")
write.table(dat, write.table(dat,
file = "results/haum/eventlogs_pre-corona_cleaned.csv", file = "results/eventlogs_pre-corona_cleaned.csv",
sep = ";", sep = ";",
quote = FALSE, quote = FALSE,
row.names = FALSE) row.names = FALSE)

10
code/06_infos-items.py
View File

@ -3,10 +3,10 @@
# content: (1) Load data and create event log # content: (1) Load data and create event log
# (2) Infos for items # (2) Infos for items
# #
# input: results/haum/eventlogs_pre-corona_cleaned.csv # input: results/eventlogs_pre-corona_cleaned.csv
# output: results/haum/pn_infos_items.csv # output: results/pn_infos_items.csv
# #
# last mod: 2024-03-06 # last mod: 2024-03-22
import pm4py import pm4py
import pandas as pd import pandas as pd
@ -16,7 +16,7 @@ from python_helpers import eval_pm, pn_infos
#--------------- (1) Load data and create event logs --------------- #--------------- (1) Load data and create event logs ---------------
dat = pd.read_csv("results/haum/eventlogs_pre-corona_cleaned", sep = ";") dat = pd.read_csv("results/eventlogs_pre-corona_cleaned", sep = ";")
log_path = pm4py.format_dataframe(dat, case_id = "path", activity_key = "event", log_path = pm4py.format_dataframe(dat, case_id = "path", activity_key = "event",
timestamp_key = "date.start") timestamp_key = "date.start")
@ -33,5 +33,5 @@ for item in log_path.item.unique().tolist():
eval = eval.sort_index() eval = eval.sort_index()
# Export # Export
eval.to_csv("results/haum/pn_infos_items.csv", sep = ";") eval.to_csv("results/pn_infos_items.csv", sep = ";")

36
code/07_item-clustering.R
View File

@ -7,12 +7,12 @@
# (2) Clustering # (2) Clustering
# (3) Visualization with pictures # (3) Visualization with pictures
# #
# input: results/haum/eventlogs_pre-corona_cleaned.RData # input: results/eventlogs_pre-corona_cleaned.RData
# results/haum/pn_infos_items.csv # results/pn_infos_items.csv
# output: results/haum/eventlogs_pre-corona_item-clusters.csv # output: results/eventlogs_pre-corona_item-clusters.csv
# ../../thesis/figures/data/clustering_items.RData" # ../../thesis/figures/data/clustering_items.RData"
# #
# last mod: 2024-03-21 # last mod: 2024-03-22
# setwd("C:/Users/nwickelmaier/Nextcloud/Documents/MDS/2023ss/60100_master_thesis/analysis/code") # setwd("C:/Users/nwickelmaier/Nextcloud/Documents/MDS/2023ss/60100_master_thesis/analysis/code")
@ -22,11 +22,11 @@ source("R_helpers.R")
#--------------- (1.1) Read log event data --------------- #--------------- (1.1) Read log event data ---------------
load("results/haum/eventlogs_pre-corona_cleaned.RData") load("results/eventlogs_pre-corona_cleaned.RData")
#--------------- (1.2) Read infos for PM for items --------------- #--------------- (1.2) Read infos for PM for items ---------------
datitem <- read.table("results/haum/pn_infos_items.csv", header = TRUE, datitem <- read.table("results/pn_infos_items.csv", header = TRUE,
sep = ";", row.names = 1) sep = ";", row.names = 1)
#--------------- (1.3) Extract additional infos for clustering --------------- #--------------- (1.3) Extract additional infos for clustering ---------------
@ -126,6 +126,28 @@ item <- sprintf("%03d", as.numeric(gsub("item_([0-9]{3})", "\\1",
res <- merge(dat, data.frame(item, cluster), by = "item", all.x = TRUE) res <- merge(dat, data.frame(item, cluster), by = "item", all.x = TRUE)
res <- res[order(res$fileId.start, res$date.start, res$timeMs.start), ] res <- res[order(res$fileId.start, res$date.start, res$timeMs.start), ]
# DFGs for clusters
res$start <- res$date.start
res$complete <- res$date.stop
for (clst in sort(unique(res$cluster))) {
alog <- bupaR::activitylog(res[res$cluster == clst, ],
case_id = "path",
activity_id = "event",
resource_id = "item",
timestamps = c("start", "complete"))
processmapR::process_map(alog,
type_nodes = processmapR::frequency("relative", color_scale = "Greys"),
sec_nodes = processmapR::frequency("absolute"),
type_edges = processmapR::frequency("relative", color_edges = "#FF6900"),
sec_edges = processmapR::frequency("absolute"),
rankdir = "LR")
}
# Look at clusters # Look at clusters
par(mfrow = c(2,2)) par(mfrow = c(2,2))
vioplot::vioplot(duration ~ cluster, res) vioplot::vioplot(duration ~ cluster, res)
@ -134,7 +156,7 @@ vioplot::vioplot(scaleSize ~ cluster, res)
vioplot::vioplot(rotationDegree ~ cluster, res) vioplot::vioplot(rotationDegree ~ cluster, res)
write.table(res, write.table(res,
file = "results/haum/eventlogs_pre-corona_item-clusters.csv", file = "results/eventlogs_pre-corona_item-clusters.csv",
sep = ";", sep = ";",
quote = FALSE, quote = FALSE,
row.names = FALSE) row.names = FALSE)

4
code/python_helpers.py
View File

@ -36,8 +36,8 @@ def pn_infos_miner(log, miner):
net, im, fm = pm4py.discover_petri_net_ilp(log) net, im, fm = pm4py.discover_petri_net_ilp(log)
elif miner == "inductive": elif miner == "inductive":
net, im, fm = pm4py.discover_petri_net_inductive(log) net, im, fm = pm4py.discover_petri_net_inductive(log)
elif miner == "conformative": elif miner == "normative":
net, im, fm = pm4py.read_pnml("results/haum/conformative_petrinet_con.pnml") net, im, fm = pm4py.read_pnml("results/normative_petrinet.pnml")
eval = eval_append(log, net, im, fm) eval = eval_append(log, net, im, fm)
eval.index = [miner] eval.index = [miner]