How to Create a Document Set in SharePoint 2013 using REST API

You can perform basic create, read, update, and delete (CRUD) operations by using the Representational State Transfer (REST) interface provided by SharePoint. The REST interface exposes all the SharePoint entities and operations that are available in the other SharePoint client APIs. One advantage of using REST is that you don't have to add references to any SharePoint libraries or client assemblies. Instead, you make HTTP requests to the appropriate endpoints to retrieve or update SharePoint entities, such as webs, lists, and list items

Source:https://docs.microsoft.com/en-us/sharepoint/dev/sp-add-ins/complete-basic-operations-using-sharepoint-rest-endpoints 

Create Document Set named DemoSet in Documents library:

$(function() 
    {
    createDocumentSet(_spPageContextInfo.webAbsoluteUrl,'Documents','DemoSet',
      function(folder){
        console.log('Document Set ' + folder.Name + ' has been created succesfully'); 
      },
      function(error){
        console.log(JSON.stringify(error));
      }
    );
    
    });
    function createDocumentSet(webUrl,listName,folderName, success, error)
    {
       createFolder(webUrl,listName,folderName,'0x0120D520', success, error);
    }
    
    function getListUrl(webUrl,listName,success, error) 
    {
        var headers = {};
        $.ajax({       
           url: webUrl + "/_api/lists/getbytitle('" + listName + "')/rootFolder?$select=ServerRelativeUrl",   
           type: "GET",   
           contentType: "application/json;odata=verbose",
           headers: { 
              "Accept": "application/json;odata=verbose"
           },   
           success: function(data){
               success(data.d.ServerRelativeUrl);
           },
           error: error
        });
    }
    
    function createFolder(webUrl,listName,folderName,folderContentTypeId, success, error) 
    {  
        getListUrl(webUrl,listName,
          function(listUrl) {
              var folderPayload = {
                 'Title' : folderName,
                 'Path' : listUrl
              };
    
              //Create Folder resource
              $.ajax({
                    url: webUrl + "/_vti_bin/listdata.svc/" + listName,
                    type: "POST",
                    contentType: "application/json;odata=verbose",
                    data: JSON.stringify(folderPayload),
                    headers: {
                       "Accept": "application/json;odata=verbose",
                       "Slug": listUrl + "/" + folderName + "|" + folderContentTypeId
                    },
                    success: function (data) {
                        success(data.d);
                    },
                    error: error
              });
          },
          error);
    }