Formulario parte GES di Business data analytics

#--------FORMULARIO modulo 2 (gestionale) --------

#NETWORK ANALYSIS - LAB1

#loading a Network

#Inspecting a Network loaded

#Descriptive statistics for the network

#Degree and Weighted Degree Distributions

#Centrality Measures

set.seed(123)

library(igraph)

library(dplyr)

library(anytime)

library("reshape2")

library(RColorBrewer)

library(ggplot2)

library(corrplot)

#loading the data

data <- read.table(

file = 'email_network.tsv',

header = FALSE,

col.names = c("sender","recipient","num_emails", "timestamp"),

skip = 2)

data$timestamp <- anytime(data$timestamp)

head(data)

summary(data)

#Weighted Network: aggregation of data to obtain a weighted network

weighted_data <- aggregate(

data$num_emails,

by=list(data$sender,data$recipient),

FUN=sum)

colnames(weighted_data) <- c("sender","recipient","num_emails")

summary(weighted_data)

mal

#big gap third quantile - max value -> mean >> median + weight distribution non

normal

#TRASFORM THE NETWORK INTO ITS WEIGHTED VERSION

#g_air = netowrk name

#Passengers = attribute according to which I want to assign a weight

E(g_air)$weight <- E(g_air)$Passengers

g_air

#Weight distribution on a logaritmic scale

hist(log10(weighted_data$num_emails))

#cumulative distribution of the weighted data

x <- sort(weighted_data$num_emails)

y <- cumsum(x)/sum(x)

plot(x,y)

#EXE1: LOADING DATA IN NETWORK FORM

#1 case: Undirected and unweighted

simple_network <- make_graph(

c(

1,2, # <- link 1

2,3, # <- link 2

3,4, # ...

1,3,

4,7,

5,6 # <- link N

), directed=FALSE)

#Connectivity

is_connected(simple_network) #TRUE or FALSE

components(simple_network) #output: nodes with the number of the component they

belong

#2 case: directed and unweighted (UN)

people_network <- graph_from_literal(

Marco-+Andrea,

Andrea-+Laura,

Marco-+Michele,

Lucia-+Andrea,

Lucia-+Laura,

Laura-+Stefania,

Lucia-+Stefania,

Michele-+Stefania)

#loading a file as a network

warehouse_network <- read_graph("warehouse_network.gml", format="graphml")

#undirected and unweighted

#constructing a network given a dataframe

#selection of UNIQUE nodes

nodes_sender = as.vector(unique(weighted_data$sender))

nodes_recipient = as.vector(unique(weighted_data$recipient))

nodes = c(nodes_sender, nodes_recipient)

nodes = sort(unique(nodes))

g = graph_from_data_frame(weighted_data, directed=T, vertices = nodes)

g #DN: Directed and Named nodes

#167 nodes and 5784 links

#second line: node attributes (name) + edge attributes (num_emails) + their types (c

= character, n = numeroc)

#third line: preview of the links

#if the network would be weighted, it appears DNW + "weight" as edge name

#MAKING A NETWOEK WEIGHTED

colnames(weighted_data)[length(colnames(weighted_data))] <- "weight"

g = graph_from_data_frame(weighted_data, directed=T, vertices = nodes)

g

#SELF-LOOPS?

any_loop(g)

#identifying self loop

E(g)[which_loop(g)]

#removing self loops

g <- simplify(g)

#saving the network

write_graph(

graph=g,

file="email_network2.gml",

format="graphml")

#verifying the saved network -> identical_graph() -> True or False?

saved_network <- read_graph("email_network2.gml", format="graphml")

identical_graphs(g, saved_network, attrs = FALSE) #as we don't want to compare

networks based on attributes

#INSPECTING THE NETWORK

#HOW MANY NODES first 10 nodes

V(g)[1:10]

#HOW MANY LINKS first 10 links

E(g)[1:10]

#weights of the first 10 links

E(g)$weight[1:10]

#How many e-mails (links) were sent from the node in position 13 to the node in

position 48?

g[13,48]

#What are the first 5 outgoing links from the node in position 8?

g[8,c(1:5)]

#What are the first 5 incoming links to the node in position 8?

g[c(1:5),8]

#What is the WEIGHTED OUTDEGREE (out-strength) of the node in position 67?

sum(g[67,])

#What is the WEIGHTED INDEGREE (in-strength) of the node in position 67?

sum(g[,67])

#Does the node names coincide with their position in the node list?

sum(g[,67]) == sum(g[,"67"])

#nodes can be indicated both with their NAME or POSITION

V(people_network)

people_network[5]

people_network['Lucia']

#DESCRIPTIVE STATISTICS

#Number of Links and Nodes

vcount(g)

ecount(g)

#Network Density

edge_density(g)

#Number of Connected Components

components(g)$no

#Diameter and Mean Shortest-Path Lenght

diameter(g)

mean_distance(g)

#Collecting Statistics into a Dataframe

descriptive <- as.data.frame(rbind(

vcount(g), ecount(g), edge_density(g),

diameter(g), components(g)$no, mean_distance(g)),

row.names = c(

"Nodes Number", "Links Number", "Density",

"Diameter","Connected components", "Average path length"))

descriptive

#question: there are 10 links with weight greater than 0.85

sum(E(g)$weight > 0.85)

#ADJACENCY MATRIX

M <- as_adjacency_matrix(g)

M_weighted <- as_adjacency_matrix(g, attr='weight')

#if small, you can display it

M_small <- as_adjacency_matrix(people_network)

M_small

#if large, use Image

par(mfrow=c(1,2))

image(as.matrix(M), zlim=c(0.5,1), xaxt='n', yaxt='n')

image(as.matrix(M_weighted),zlim=c(1,30), #limit color range

col=hcl.colors(12, "YlOrRd"), xaxt='n', yaxt='n')

#Network Visualization

par(mfrow=c(1,2))

plot(people_network)

plot(g,

layout=layout_with_fr,

vertex.size=3 ,vertex.label.dist=0.5, vertex.color="red",

edge.arrow.size=0.5)

#DEGREE DISTRIBUTION

#Visualization of the number of relationships in which a node is involved ->

distribution of the network's degree and weighted degree

#INDEGREE & OUTDEGREE

indegree <- degree(g, mode='in')

outdegree <- degree(g, mode='out')

#TOTAL DEGREE

degree <- degree(g, mode='all')

#Degree computed on the undirected version of the network

degree_U <- degree(as_undirected(g), mode='all')

#collecting the results into a dataframe

distr_degree <- as.data.frame(cbind(degree, indegree, outdegree, degree_U))

summary(distr_degree) #min, 1st Q, Median, Mean, 2rd Q, Max

#DEGREE OF A DIRECTED NETWORK

all(indegree+outdegree==degree) #true

sum(indegree)==sum(outdegree) & sum(indegree)==ecount(g) #true #somma Kin =

somma Kout = E

#DEGREE NORMALIZATION ->

degree <- degree(g, normalized = TRUE)

indegree <- degree(g, mode='in', normalized = TRUE)

outdegree <- degree(g, mode='out', normalized = TRUE)

distr_degree_norm <- as.data.frame(cbind(degree, indegree, outdegree))

distr_degree_norm$degree <- distr_degree_norm$degree/2

distr_degree_norm$degree_U <- degree(as_undirected(g), normalized = TRUE)

summary(distr_degree_norm)

summary(distr_degree_norm)

#ORDERING THE 10 TOP NODES BY CENTRALITY 2 MEASURES (USING INDEGREE AND

INSTRENGHT). iNTERSECTION

indegree <- degree(g_air, mode='in')

instrength <- strength(g_air, mode='in')

top10_indegree <- names(indegree[order(indegree, decreasing=T)][1:10])

top10_instrength <- names(instrength[order(instrength, decreasing=T)][1:10])

intersect(top10_indegree, top10_instrength)

#from which cities do the most central airports originate?

V(g_air)[intersect(top10_indegree, top10_instrength)]$City

#average shortest-path length

mean_distance(g_airNew)

#QUANDO CHIEDONO MEDIA DEL GRADO PESATO CHIEDONO LA STRENGHT

#AS the normalized option does not distinguish between

#directed and undirected networks and always applies the same normalisation factor

#(� − 1). As a consequence, the normalised degree ranges between 0 and 2.

#WEIGHTED DEGREE DISTRIBUTION

strength <- strength(g, mode='all')

instrength <- strength(g, mode='in')

outstrength <- strength(g, mode='out')

distr_strength <- as.data.frame(cbind(strength, instrength, outstrength))

summary(distr_strength)

#STRENGHT NORMALIZATION NECESSARY AS it doesn't exist a theorical maximum

value of node strenght, it depends on the observed data

M <- max(max(distr_strength$outstrength), max(distr_strength$instrength))

distr_strength_norm <- distr_strength

distr_strength_norm$strength <- distr_strength_norm$strength/(2*M) #as stenght is

the sum of instrenght and outstrenght

distr_strength_norm$instrength <- distr_strength_norm$instrength/M

distr_strength_norm$outstrength <- distr_strength_norm$outstrength/M

summary(distr_strength_norm) #now the three distributions are comparable

#COMPARING NODE POSITIONS

#IDENTIFY THE 20 MOST CENTRAL NODES ACCORDING TO THESE MEASURES:

CLOSENESS CENTRALITY, BETWEENNESS, UNDIRECTED STRENGTH

stren <- strength(g)

betw <- betweenness(g)

close <- closeness(g)

# Top20

top_stren <- names(sort(stren, decreasing = TRUE))[1:20]

top_betw <- names(sort(betw, decreasing = TRUE))[1:20]

top_close <- names(sort(close, decreasing = TRUE))[1:20]

top_stren <- order(stren, decreasing = TRUE)[1:20]

top_betw <- order(betw, decreasing = TRUE)[1:20]

top_close <- order(close, decreasing = TRUE)[1:20]

#Strength and betweenness share more than 3 nodes among their top-20 most

central nodes.

intersection_stren_betw <- length(intersect(top_stren, top_betw))

intersection_stren_betw > 3

#Investigation how node positions change when we consider link weights

#Mergint the degree and strenght datasets using row identifiers (node names)

all_measures <- merge(

distr_degree_norm, distr_strength_norm, by="row.names", all=T)

row.names(all_measures) <- all_measures$Row.names

all_measures$Row.names <- NULL

#converting values into ranks

all_measures_r <- as.data.frame(

apply(all_measures, FUN=rank, MARGIN=c(2)))

#rank() sostituisce ogni valore numerico con la sua posizione in classifica (dal più

piccolo al più grande).

#INSPECTING / ORDERING THE FIRST 5 NODES

#Ordered by DEGREE

row.names(all_measures_r[

order(all_measures_r[,'degree'], decreasing=T),][1:5,])

#ordered by Strength

row.names(all_measures_r[

order(all_measures_r[,'strength'], decreasing=T),][1:5,])

#10 nodes by each degree-based measure

#degree

row.names(all_measures_r[

order(all_measures_r[,'degree'], decreasing=T),][1:10,])

#indegree

row.names(all_measures_r[

order(all_measures_r[,'indegree'], decreasing=T),][1:10,])

#outdegree

row.names(all_measures_r[

order(all_measures_r[,'outdegree'], decreasing=T),][1:10,])

#strenght

row.names(all_measures_r[

order(all_measures_r[,'strength'], decreasing=T),][1:10,])

#instrength

row.names(all_measures_r[

order(all_measures_r[,'instrength'], decreasing=T),][1:10,])

#outstrength

row.names(all_measures_r[

order(all_measures_r[,'outstrength'], decreasing=T),][1:10,])

#if central nodes differ substantially across measures a specialisation of node roles is

suggested

#NODES IN COMMON ACROSS THE DIFFERENT DEGREE MEASURES

d1 <- row.names(all_measures_r[

order(all_measures_r[,'degree'], decreasing=T),][1:10,])

d2 <- row.names(all_measures_r[

order(all_measures_r[,'indegree'], decreasing=T),][1:10,])

d3 <- row.names(all_measures_r[

order(all_measures_r[,'outdegree'], decreasing=T),][1:10,])

intersect(intersect(d1,d2),d3)

#NODES IN COMMON ACROSS THE DIFFERENT STRENGHT MEASURES

s1 <- row.names(all_measures_r[

order(all_measures_r[,'strength'], decreasing=T),][1:10,])

s2 <- row.names(all_measures_r[

order(all_measures_r[,'instrength'], decreasing=T),][1:10,])

s3 <- row.names(all_measures_r[

order(all_measures_r[,'outstrength'], decreasing=T),][1:10,])

intersect(intersect(s1,s2),s3)

#NODE THAT BELONGS TO THE TOP 10 POSITIONS BOTH IN DEGREE AND STRENGHT

i1 <- intersect(intersect(d1,d2),d3)

i2 <- intersect(intersect(s1,s2),s3)

intersect(i1,i2)

#IDENTIFY THE 8 MOST CENTRAL MODES ACCORDING TO CLOSENESS CENTRALITY,

AUTHORITY SCORE, STRENGTH

c1 <- as.data.frame(closeness(g))

c2 <- as.data.frame(authority.score(g)$vector)

c3 <- as.data.frame(strength(g))

c1$id <- row.names(c1)

c2$id <- row.names(c2)

c3$id <- row.names(c3)

top8c1 <- c1[

order(c1[[1]], decreasing=T),"id"][1:8] #veryfing here if specific nodes are there

top8c2 <- c2[

order(c2[[1]], decreasing=T),"id"][1:8]

top8c3 <- c3[

order(c3[[1]], decreasing=T),"id"][1:8]

#nodes in commont with all the 3 measures

length(intersect(intersect(top8c1,top8c2),top8c3))

#number of nodes in common between the top 8 nodes by strength and closeness

#is greater than the number of nodes in common between the top 8 nodes by

strength and authority score.

length(intersect(top8c1, top8c3)) >= length(intersect(top8c3, top8c2))

#CORRELATION MATRIX NODE RANKING -> Kendall's Method (we are correlating ranks)

#how different the node rankings are across measures, we compute correlation

coefficients

round(cor(all_measures_r, method='kendall'),3)

#are correlations statistically significant?

options(width=70)

cor.test(

all_measures_r[['degree']],

all_measures_r[['outstrength']],

method='kendall')

#OPPURE QUANDO SI HANNO SOLO 2 CENTRALITIES

cor(rank(indegree), rank(instrength), method = 'kendall')

#CENTRALITY MEASURES

#Normalized Degree (between 0 and 1)

degree <- degree(g, normalized = TRUE)

indegree <- degree(g, mode='in', normalized = TRUE)

outdegree <- degree(g, mode='out', normalized = TRUE)

distr_degree_norm <- as.data.frame(cbind(degree, indegree, outdegree))

distr_degree_norm$degree <- distr_degree_norm$degree/2

#Normalized strenght

strength <- strength(g, mode='all')

instrength <- strength(g, mode='in')

outstrength <- strength(g, mode='out')

distr_strength <- as.data.frame(cbind(strength, instrength, outstrength))

M <- max(max(distr_strength$outstrength), max(distr_strength$instrength))

distr_strength_norm <- distr_strength

distr_strength_norm$strength <- distr_strength_norm$strength/(2*M)

distr_strength_norm$instrength <- distr_strength_norm$instrength/M

distr_strength_norm$outstrength <- distr_strength_norm$outstrength/M

#BETWEENNESS

betweenness_g <- as.data.frame(betweenness(g, directed=T, normalized=T))

#CLOSENESS

closeness_g <- as.data.frame(closeness(g, mode='all', normalized=T))

#EIGENVECTOR

eigenvector_g <- as.data.frame(

eigen_centrality(g, directed=F, scale=T)$vector)

#PAGERANK & AUTHORITY SCORE & HUB SCORE

page_rank_g <- as.data.frame(page.rank(g, directed=T)$vector)

authority_score_g <- as.data.frame(authority.score(g, scale=T)$vector)

hub_score_g <- as.data.frame(hub.score(g, scale=T)$vector)

#KCORE

kcore_g <- as.data.frame(coreness(g, mode='all'))

#AVG NEAREST NEIGHBOUR DEGREE

avg_nearest_neighbour_degree_g <- as.data.frame(knn(g)$knn)

#CLUSTERING COEFFICIENT

clustering_coefficient_g <- as.data.frame(

transitivity(g, isolates="zero", type="local"),

row.names=V(g)$name,

)

#use select to preserve node names on the rows

degree <- select(distr_degree_norm,'degree')

strength <- select(distr_strength_norm,'strength')

#UNWEIGHTED VERSION OF AN ALGORITHM

page.rank(g, directed=T, weights=NA)$vector

#SCALE VS NORMALIZED

#The scale option option rescales centrality values to the interval [0,1], whereas the

#normalized option (for betweenness and closeness) normalises values with respect to

a

#theoretical maximum. Therefore the resulting values are not necessarily within [0,1]:

max(hub.score(g, scale=T)$vector) == 1 #TRUE

max(betweenness(g, directed=T, normalized=T)) == 1 #FALSE

#All algorithms return node names as row identifiers except the clustering coefficient.

For

#this reason, we specify row.names=V(g)$name in the function as.data.frame.

#MERGE COMMAND - REDUCE FUNCTION

#The function merge command is a binary function that merges two datasets x amd y

#based on a shared column (in this case id). We can apply it to a list of datasets (our

centralities) via the function Reduce:

#store all datasets in a single list

all_centralities <- list(

degree, strength, betweenness_g,

closeness_g, eigenvector_g, page_rank_g, authority_score_g, hub_score_g,

kcore_g, avg_nearest_neighbour_degree_g, clustering_coefficient_g)

# add an id column to merge datasets

for (i in 1:length(all_centralities)){

all_centralities[[i]]$id <- row.names(all_centralities[[i]])}

# apply merge across all datasets

all_centralities_df <- Reduce(

function(x, y) merge(x, y, all=TRUE, by="id"), all_centralities)

# rename columns for space reasons

colnames(all_centralities_df) <- c(

"id", "d", "s", "b", "c", "e", "p", "a", "h", "k", "knn", "cc")

#differences across the top 10 positions in the network

top10_all <- list()

# start from 2 because the first column is "id"

for (c in 2:length(colnames(all_centralities_df))){

cc <- colnames(all_centralities_df)[c]

top10_all[[cc]] <- all_centralities_df[

order(all_centralities_df[,c], decreasing=T),][1:10,]$id

}

#top 10 positions from single vectors / single centrality measure-> we must first

create an ID COLUMN for the node names

eigenvector_g <- as.data.frame(

eigen_centrality(g, directed=F, scale=T)$vector