The problem is simply that you are getting the wrong values out of your predict call, because by default it sets the return type to type = "link", whereas you are looking for type = "response". If you make this change, you get identical results to ggplot, which knows to use type = "response" without being asked:

data$glm_nb <- predict(ans, type = "response")

ggplot(data, aes(x = times, y = value, group = variable)) +
  geom_point(size=2,alpha = 0.2) +
  stat_smooth(method = "glm.nb", 
              formula = y ~ splines::bs(x, Boundary.knots = c(0,10), 
                                        knots = c(3), degree = 3, 
                                        intercept = FALSE), 
              color = "green", size = 0.3) +
  geom_line(aes(x = times, y = glm_nb), color = "blue") +
  facet_grid(.~variable) +
  labs(x = "times", y = "Value") +
  theme_bw(base_size = 22)