#include #define GGML_MAX_NAME 128 #include #include #include #include #include #include #include "gosd.h" #define STB_IMAGE_IMPLEMENTATION #define STB_IMAGE_STATIC #include "stb_image.h" #define STB_IMAGE_WRITE_IMPLEMENTATION #define STB_IMAGE_WRITE_STATIC #include "stb_image_write.h" #define STB_IMAGE_RESIZE_IMPLEMENTATION #define STB_IMAGE_RESIZE_STATIC #include "stb_image_resize.h" // Names of the sampler method, same order as enum sample_method in stable-diffusion.h const char* sample_method_str[] = { "default", "euler", "heun", "dpm2", "dpm++2s_a", "dpm++2m", "dpm++2mv2", "ipndm", "ipndm_v", "lcm", "ddim_trailing", "tcd", "euler_a", }; static_assert(std::size(sample_method_str) == SAMPLE_METHOD_COUNT, "sample method mismatch"); // Names of the sigma schedule overrides, same order as sample_schedule in stable-diffusion.h const char* schedulers[] = { "default", "discrete", "karras", "exponential", "ays", "gits", "smoothstep", }; static_assert(std::size(schedulers) == SCHEDULE_COUNT, "schedulers mismatch"); sd_ctx_t* sd_c; // Moved from the context (load time) to generation time params scheduler_t scheduler = scheduler_t::DEFAULT; sample_method_t sample_method; // Copied from the upstream CLI static void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) { //SDParams* params = (SDParams*)data; const char* level_str; if (!log /*|| (!params->verbose && level <= SD_LOG_DEBUG)*/) { return; } switch (level) { case SD_LOG_DEBUG: level_str = "DEBUG"; break; case SD_LOG_INFO: level_str = "INFO"; break; case SD_LOG_WARN: level_str = "WARN"; break; case SD_LOG_ERROR: level_str = "ERROR"; break; default: /* Potential future-proofing */ level_str = "?????"; break; } fprintf(stderr, "[%-5s] ", level_str); fputs(log, stderr); fflush(stderr); } int load_model(const char *model, char *model_path, char* options[], int threads, int diff) { fprintf (stderr, "Loading model: %p=%s\n", model, model); sd_set_log_callback(sd_log_cb, NULL); const char *stableDiffusionModel = ""; if (diff == 1 ) { stableDiffusionModel = model; model = ""; } // decode options. Options are in form optname:optvale, or if booleans only optname. const char *clip_l_path = ""; const char *clip_g_path = ""; const char *t5xxl_path = ""; const char *vae_path = ""; const char *scheduler_str = ""; const char *sampler = ""; char *lora_dir = model_path; bool lora_dir_allocated = false; fprintf(stderr, "parsing options: %p\n", options); // If options is not NULL, parse options for (int i = 0; options[i] != NULL; i++) { const char *optname = strtok(options[i], ":"); const char *optval = strtok(NULL, ":"); if (optval == NULL) { optval = "true"; } if (!strcmp(optname, "clip_l_path")) { clip_l_path = optval; } if (!strcmp(optname, "clip_g_path")) { clip_g_path = optval; } if (!strcmp(optname, "t5xxl_path")) { t5xxl_path = optval; } if (!strcmp(optname, "vae_path")) { vae_path = optval; } if (!strcmp(optname, "scheduler")) { scheduler_str = optval; } if (!strcmp(optname, "sampler")) { sampler = optval; } if (!strcmp(optname, "lora_dir")) { // Path join with model dir if (model_path && strlen(model_path) > 0) { std::filesystem::path model_path_str(model_path); std::filesystem::path lora_path(optval); std::filesystem::path full_lora_path = model_path_str / lora_path; lora_dir = strdup(full_lora_path.string().c_str()); lora_dir_allocated = true; fprintf(stderr, "Lora dir resolved to: %s\n", lora_dir); } else { lora_dir = strdup(optval); lora_dir_allocated = true; fprintf(stderr, "No model path provided, using lora dir as-is: %s\n", lora_dir); } } } fprintf(stderr, "parsed options\n"); int sample_method_found = -1; for (int m = 0; m < SAMPLE_METHOD_COUNT; m++) { if (!strcmp(sampler, sample_method_str[m])) { sample_method_found = m; fprintf(stderr, "Found sampler: %s\n", sampler); } } if (sample_method_found == -1) { fprintf(stderr, "Invalid sample method, default to EULER_A!\n"); sample_method_found = sample_method_t::SAMPLE_METHOD_DEFAULT; } sample_method = (sample_method_t)sample_method_found; for (int d = 0; d < SCHEDULE_COUNT; d++) { if (!strcmp(scheduler_str, schedulers[d])) { scheduler = (scheduler_t)d; fprintf (stderr, "Found scheduler: %s\n", scheduler_str); } } fprintf (stderr, "Creating context\n"); sd_ctx_params_t ctx_params; sd_ctx_params_init(&ctx_params); ctx_params.model_path = model; ctx_params.clip_l_path = clip_l_path; ctx_params.clip_g_path = clip_g_path; ctx_params.t5xxl_path = t5xxl_path; ctx_params.diffusion_model_path = stableDiffusionModel; ctx_params.vae_path = vae_path; ctx_params.taesd_path = ""; ctx_params.control_net_path = ""; ctx_params.lora_model_dir = lora_dir; ctx_params.embedding_dir = ""; ctx_params.vae_decode_only = false; ctx_params.free_params_immediately = false; ctx_params.n_threads = threads; ctx_params.rng_type = STD_DEFAULT_RNG; sd_ctx_t* sd_ctx = new_sd_ctx(&ctx_params); if (sd_ctx == NULL) { fprintf (stderr, "failed loading model (generic error)\n"); // Clean up allocated memory if (lora_dir_allocated && lora_dir) { free(lora_dir); } return 1; } fprintf (stderr, "Created context: OK\n"); sd_c = sd_ctx; // Clean up allocated memory if (lora_dir_allocated && lora_dir) { free(lora_dir); } return 0; } void sd_tiling_params_set_enabled(sd_tiling_params_t *params, bool enabled) { params->enabled = enabled; } void sd_tiling_params_set_tile_sizes(sd_tiling_params_t *params, int tile_size_x, int tile_size_y) { params->tile_size_x = tile_size_x; params->tile_size_y = tile_size_y; } void sd_tiling_params_set_rel_sizes(sd_tiling_params_t *params, float rel_size_x, float rel_size_y) { params->rel_size_x = rel_size_x; params->rel_size_y = rel_size_y; } void sd_tiling_params_set_target_overlap(sd_tiling_params_t *params, float target_overlap) { params->target_overlap = target_overlap; } sd_tiling_params_t* sd_img_gen_params_get_vae_tiling_params(sd_img_gen_params_t *params) { return ¶ms->vae_tiling_params; } sd_img_gen_params_t* sd_img_gen_params_new(void) { sd_img_gen_params_t *params = (sd_img_gen_params_t *)std::malloc(sizeof(sd_img_gen_params_t)); sd_img_gen_params_init(params); return params; } void sd_img_gen_params_set_prompts(sd_img_gen_params_t *params, const char *prompt, const char *negative_prompt) { params->prompt = prompt; params->negative_prompt = negative_prompt; } void sd_img_gen_params_set_dimensions(sd_img_gen_params_t *params, int width, int height) { params->width = width; params->height = height; } void sd_img_gen_params_set_seed(sd_img_gen_params_t *params, int64_t seed) { params->seed = seed; } int gen_image(sd_img_gen_params_t *p, int steps, char *dst, float cfg_scale, char *src_image, float strength, char *mask_image, char **ref_images, int ref_images_count) { sd_image_t* results; std::vector skip_layers = {7, 8, 9}; fprintf (stderr, "Generating image\n"); p->sample_params.guidance.txt_cfg = cfg_scale; p->sample_params.guidance.slg.layers = skip_layers.data(); p->sample_params.guidance.slg.layer_count = skip_layers.size(); p->sample_params.sample_method = sample_method; p->sample_params.sample_steps = steps; p->sample_params.scheduler = scheduler; int width = p->width; int height = p->height; // Handle input image for img2img bool has_input_image = (src_image != NULL && strlen(src_image) > 0); bool has_mask_image = (mask_image != NULL && strlen(mask_image) > 0); uint8_t* input_image_buffer = NULL; uint8_t* mask_image_buffer = NULL; std::vector default_mask_image_vec; if (has_input_image) { fprintf(stderr, "Loading input image: %s\n", src_image); int c = 0; int img_width = 0; int img_height = 0; input_image_buffer = stbi_load(src_image, &img_width, &img_height, &c, 3); if (input_image_buffer == NULL) { fprintf(stderr, "Failed to load input image from '%s'\n", src_image); return 1; } if (c < 3) { fprintf(stderr, "Input image must have at least 3 channels, got %d\n", c); free(input_image_buffer); return 1; } // Resize input image if dimensions don't match if (img_width != width || img_height != height) { fprintf(stderr, "Resizing input image from %dx%d to %dx%d\n", img_width, img_height, width, height); uint8_t* resized_image_buffer = (uint8_t*)malloc(height * width * 3); if (resized_image_buffer == NULL) { fprintf(stderr, "Failed to allocate memory for resized image\n"); free(input_image_buffer); return 1; } stbir_resize(input_image_buffer, img_width, img_height, 0, resized_image_buffer, width, height, 0, STBIR_TYPE_UINT8, 3, STBIR_ALPHA_CHANNEL_NONE, 0, STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP, STBIR_FILTER_BOX, STBIR_FILTER_BOX, STBIR_COLORSPACE_SRGB, nullptr); free(input_image_buffer); input_image_buffer = resized_image_buffer; } p->init_image = {(uint32_t)width, (uint32_t)height, 3, input_image_buffer}; p->strength = strength; fprintf(stderr, "Using img2img with strength: %.2f\n", strength); } else { // No input image, use empty image for text-to-image p->init_image = {(uint32_t)width, (uint32_t)height, 3, NULL}; p->strength = 0.0f; } // Handle mask image for inpainting if (has_mask_image) { fprintf(stderr, "Loading mask image: %s\n", mask_image); int c = 0; int mask_width = 0; int mask_height = 0; mask_image_buffer = stbi_load(mask_image, &mask_width, &mask_height, &c, 1); if (mask_image_buffer == NULL) { fprintf(stderr, "Failed to load mask image from '%s'\n", mask_image); if (input_image_buffer) free(input_image_buffer); return 1; } // Resize mask if dimensions don't match if (mask_width != width || mask_height != height) { fprintf(stderr, "Resizing mask image from %dx%d to %dx%d\n", mask_width, mask_height, width, height); uint8_t* resized_mask_buffer = (uint8_t*)malloc(height * width); if (resized_mask_buffer == NULL) { fprintf(stderr, "Failed to allocate memory for resized mask\n"); free(mask_image_buffer); if (input_image_buffer) free(input_image_buffer); return 1; } stbir_resize(mask_image_buffer, mask_width, mask_height, 0, resized_mask_buffer, width, height, 0, STBIR_TYPE_UINT8, 1, STBIR_ALPHA_CHANNEL_NONE, 0, STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP, STBIR_FILTER_BOX, STBIR_FILTER_BOX, STBIR_COLORSPACE_SRGB, nullptr); free(mask_image_buffer); mask_image_buffer = resized_mask_buffer; } p->mask_image = {(uint32_t)width, (uint32_t)height, 1, mask_image_buffer}; fprintf(stderr, "Using inpainting with mask\n"); } else { // No mask image, create default full mask default_mask_image_vec.resize(width * height, 255); p->mask_image = {(uint32_t)width, (uint32_t)height, 1, default_mask_image_vec.data()}; } // Handle reference images std::vector ref_images_vec; std::vector ref_image_buffers; if (ref_images_count > 0 && ref_images != NULL) { fprintf(stderr, "Loading %d reference images\n", ref_images_count); for (int i = 0; i < ref_images_count; i++) { if (ref_images[i] == NULL || strlen(ref_images[i]) == 0) { continue; } fprintf(stderr, "Loading reference image %d: %s\n", i + 1, ref_images[i]); int c = 0; int ref_width = 0; int ref_height = 0; uint8_t* ref_image_buffer = stbi_load(ref_images[i], &ref_width, &ref_height, &c, 3); if (ref_image_buffer == NULL) { fprintf(stderr, "Failed to load reference image from '%s'\n", ref_images[i]); continue; } if (c < 3) { fprintf(stderr, "Reference image must have at least 3 channels, got %d\n", c); free(ref_image_buffer); continue; } // Resize reference image if dimensions don't match if (ref_width != width || ref_height != height) { fprintf(stderr, "Resizing reference image from %dx%d to %dx%d\n", ref_width, ref_height, width, height); uint8_t* resized_ref_buffer = (uint8_t*)malloc(height * width * 3); if (resized_ref_buffer == NULL) { fprintf(stderr, "Failed to allocate memory for resized reference image\n"); free(ref_image_buffer); continue; } stbir_resize(ref_image_buffer, ref_width, ref_height, 0, resized_ref_buffer, width, height, 0, STBIR_TYPE_UINT8, 3, STBIR_ALPHA_CHANNEL_NONE, 0, STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP, STBIR_FILTER_BOX, STBIR_FILTER_BOX, STBIR_COLORSPACE_SRGB, nullptr); free(ref_image_buffer); ref_image_buffer = resized_ref_buffer; } ref_image_buffers.push_back(ref_image_buffer); ref_images_vec.push_back({(uint32_t)width, (uint32_t)height, 3, ref_image_buffer}); } if (!ref_images_vec.empty()) { p->ref_images = ref_images_vec.data(); p->ref_images_count = ref_images_vec.size(); fprintf(stderr, "Using %zu reference images\n", ref_images_vec.size()); } } results = generate_image(sd_c, p); std::free(p); if (results == NULL) { fprintf (stderr, "NO results\n"); if (input_image_buffer) free(input_image_buffer); if (mask_image_buffer) free(mask_image_buffer); for (auto buffer : ref_image_buffers) { if (buffer) free(buffer); } return 1; } if (results[0].data == NULL) { fprintf (stderr, "Results with no data\n"); if (input_image_buffer) free(input_image_buffer); if (mask_image_buffer) free(mask_image_buffer); for (auto buffer : ref_image_buffers) { if (buffer) free(buffer); } return 1; } fprintf (stderr, "Writing PNG\n"); fprintf (stderr, "DST: %s\n", dst); fprintf (stderr, "Width: %d\n", results[0].width); fprintf (stderr, "Height: %d\n", results[0].height); fprintf (stderr, "Channel: %d\n", results[0].channel); fprintf (stderr, "Data: %p\n", results[0].data); stbi_write_png(dst, results[0].width, results[0].height, results[0].channel, results[0].data, 0, NULL); fprintf (stderr, "Saved resulting image to '%s'\n", dst); // Clean up free(results[0].data); results[0].data = NULL; free(results); if (input_image_buffer) free(input_image_buffer); if (mask_image_buffer) free(mask_image_buffer); for (auto buffer : ref_image_buffers) { if (buffer) free(buffer); } fprintf (stderr, "gen_image is done: %s", dst); return 0; } int unload() { free_sd_ctx(sd_c); return 0; }