Perfect multicollinearity
set.seed(1)
X <- matrix(sample(c(0,1,2), 15, rep=TRUE), ncol=3)
X
## [,1] [,2] [,3]
## [1,] 0 2 0
## [2,] 1 2 0
## [3,] 1 1 2
## [4,] 2 1 1
## [5,] 0 0 2
det(t(X)%*%X)
## [1] 221
X[,3] <- X[,1] + 2*X[,2]
X
## [,1] [,2] [,3]
## [1,] 0 2 4
## [2,] 1 2 5
## [3,] 1 1 3
## [4,] 2 1 4
## [5,] 0 0 0
det(t(X)%*%X)
## [1] 0
Ridge regression - the role of lambda
lambda1 <- 1
det(t(X) %*% X + diag(lambda1,3))
## [1] 293
set.seed(1)
lambda2 <- 0.1
X <- matrix(sample(c(0,1,2), 15, replace=TRUE), nrow=3, ncol=5)
X
## [,1] [,2] [,3] [,4] [,5]
## [1,] 0 2 2 0 2
## [2,] 1 0 1 0 1
## [3,] 1 2 1 0 2
det(t(X) %*% X)
## [1] 0
det(t(X) %*% X + diag(lambda2,5))
## [1] 0.33651