diff options
Diffstat (limited to 'drivers/gpu/drm/ttm/ttm_bo.c')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 290 | 
1 files changed, 102 insertions, 188 deletions
| diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index eb4b7df02ca0..9a03c7834b1e 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -45,7 +45,7 @@  static void ttm_bo_global_kobj_release(struct kobject *kobj); -/** +/*   * ttm_global_mutex - protecting the global BO state   */  DEFINE_MUTEX(ttm_global_mutex); @@ -115,10 +115,7 @@ static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo,  	struct ttm_bo_device *bdev = bo->bdev;  	struct ttm_resource_manager *man; -	if (!list_empty(&bo->lru)) -		return; - -	if (mem->placement & TTM_PL_FLAG_NO_EVICT) +	if (!list_empty(&bo->lru) || bo->pin_count)  		return;  	man = ttm_manager_type(bdev, mem->mem_type); @@ -165,7 +162,7 @@ void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,  	ttm_bo_del_from_lru(bo);  	ttm_bo_add_mem_to_lru(bo, &bo->mem); -	if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { +	if (bulk && !bo->pin_count) {  		switch (bo->mem.mem_type) {  		case TTM_PL_TT:  			ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo); @@ -234,7 +231,8 @@ EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);  static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,  				  struct ttm_resource *mem, bool evict, -				  struct ttm_operation_ctx *ctx) +				  struct ttm_operation_ctx *ctx, +				  struct ttm_place *hop)  {  	struct ttm_bo_device *bdev = bo->bdev;  	struct ttm_resource_manager *old_man = ttm_manager_type(bdev, bo->mem.mem_type); @@ -255,49 +253,20 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,  		if (ret)  			goto out_err; -		ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); -		if (ret) -			goto out_err; -  		if (mem->mem_type != TTM_PL_SYSTEM) { -			ret = ttm_tt_populate(bdev, bo->ttm, ctx); -			if (ret) -				goto out_err; - -			ret = ttm_bo_tt_bind(bo, mem); +			ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);  			if (ret)  				goto out_err;  		} - -		if (bo->mem.mem_type == TTM_PL_SYSTEM) { -			if (bdev->driver->move_notify) -				bdev->driver->move_notify(bo, evict, mem); -			bo->mem = *mem; -			goto moved; -		}  	} -	if (bdev->driver->move_notify) -		bdev->driver->move_notify(bo, evict, mem); - -	if (old_man->use_tt && new_man->use_tt) -		ret = ttm_bo_move_ttm(bo, ctx, mem); -	else if (bdev->driver->move) -		ret = bdev->driver->move(bo, evict, ctx, mem); -	else -		ret = ttm_bo_move_memcpy(bo, ctx, mem); - +	ret = bdev->driver->move(bo, evict, ctx, mem, hop);  	if (ret) { -		if (bdev->driver->move_notify) { -			swap(*mem, bo->mem); -			bdev->driver->move_notify(bo, false, mem); -			swap(*mem, bo->mem); -		} - +		if (ret == -EMULTIHOP) +			return ret;  		goto out_err;  	} -moved:  	ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;  	return 0; @@ -309,7 +278,7 @@ out_err:  	return ret;  } -/** +/*   * Call bo::reserved.   * Will release GPU memory type usage on destruction.   * This is the place to put in driver specific hooks to release @@ -319,8 +288,8 @@ out_err:  static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)  { -	if (bo->bdev->driver->move_notify) -		bo->bdev->driver->move_notify(bo, false, NULL); +	if (bo->bdev->driver->delete_mem_notify) +		bo->bdev->driver->delete_mem_notify(bo);  	ttm_bo_tt_destroy(bo);  	ttm_resource_free(bo, &bo->mem); @@ -383,9 +352,10 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)   * Must be called with lru_lock and reservation held, this function   * will drop the lru lock and optionally the reservation lock before returning.   * - * @interruptible         Any sleeps should occur interruptibly. - * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead. - * @unlock_resv           Unlock the reservation lock as well. + * @bo:                    The buffer object to clean-up + * @interruptible:         Any sleeps should occur interruptibly. + * @no_wait_gpu:           Never wait for gpu. Return -EBUSY instead. + * @unlock_resv:           Unlock the reservation lock as well.   */  static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, @@ -451,7 +421,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,  	return 0;  } -/** +/*   * Traverse the delayed list, and call ttm_bo_cleanup_refs on all   * encountered buffers.   */ @@ -540,12 +510,12 @@ static void ttm_bo_release(struct kref *kref)  		spin_lock(&ttm_bo_glob.lru_lock);  		/* -		 * Make NO_EVICT bos immediately available to +		 * Make pinned bos immediately available to  		 * shrinkers, now that they are queued for  		 * destruction.  		 */ -		if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { -			bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; +		if (bo->pin_count) { +			bo->pin_count = 0;  			ttm_bo_del_from_lru(bo);  			ttm_bo_add_mem_to_lru(bo, &bo->mem);  		} @@ -601,8 +571,11 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo,  	struct ttm_bo_device *bdev = bo->bdev;  	struct ttm_resource evict_mem;  	struct ttm_placement placement; +	struct ttm_place hop;  	int ret = 0; +	memset(&hop, 0, sizeof(hop)); +  	dma_resv_assert_held(bo->base.resv);  	placement.num_placement = 0; @@ -631,8 +604,9 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo,  		goto out;  	} -	ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx); +	ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx, &hop);  	if (unlikely(ret)) { +		WARN(ret == -EMULTIHOP, "Unexpected multihop in eviction - likely driver bug\n");  		if (ret != -ERESTARTSYS)  			pr_err("Buffer eviction failed\n");  		ttm_resource_free(bo, &evict_mem); @@ -655,7 +629,7 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,  }  EXPORT_SYMBOL(ttm_bo_eviction_valuable); -/** +/*   * Check the target bo is allowable to be evicted or swapout, including cases:   *   * a. if share same reservation object with ctx->resv, have assumption @@ -672,7 +646,7 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,  	if (bo->base.resv == ctx->resv) {  		dma_resv_assert_held(bo->base.resv); -		if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT) +		if (ctx->allow_res_evict)  			ret = true;  		*locked = false;  		if (busy) @@ -794,7 +768,7 @@ int ttm_mem_evict_first(struct ttm_bo_device *bdev,  	return ret;  } -/** +/*   * Add the last move fence to the BO and reserve a new shared slot.   */  static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, @@ -830,7 +804,7 @@ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,  	return 0;  } -/** +/*   * Repeatedly evict memory from the LRU for @mem_type until we create enough   * space, or we've evicted everything and there isn't enough space.   */ @@ -860,35 +834,11 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,  	return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);  } -static uint32_t ttm_bo_select_caching(struct ttm_resource_manager *man, -				      uint32_t cur_placement, -				      uint32_t proposed_placement) -{ -	uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; -	uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; - -	/** -	 * Keep current caching if possible. -	 */ - -	if ((cur_placement & caching) != 0) -		result |= (cur_placement & caching); -	else if ((TTM_PL_FLAG_CACHED & caching) != 0) -		result |= TTM_PL_FLAG_CACHED; -	else if ((TTM_PL_FLAG_WC & caching) != 0) -		result |= TTM_PL_FLAG_WC; -	else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) -		result |= TTM_PL_FLAG_UNCACHED; - -	return result; -} -  /**   * ttm_bo_mem_placement - check if placement is compatible   * @bo: BO to find memory for   * @place: where to search   * @mem: the memory object to fill in - * @ctx: operation context   *   * Check if placement is compatible and fill in mem structure.   * Returns -EBUSY if placement won't work or negative error code. @@ -896,23 +846,17 @@ static uint32_t ttm_bo_select_caching(struct ttm_resource_manager *man,   */  static int ttm_bo_mem_placement(struct ttm_buffer_object *bo,  				const struct ttm_place *place, -				struct ttm_resource *mem, -				struct ttm_operation_ctx *ctx) +				struct ttm_resource *mem)  {  	struct ttm_bo_device *bdev = bo->bdev;  	struct ttm_resource_manager *man; -	uint32_t cur_flags = 0;  	man = ttm_manager_type(bdev, place->mem_type);  	if (!man || !ttm_resource_manager_used(man))  		return -EBUSY; -	cur_flags = ttm_bo_select_caching(man, bo->mem.placement, -					  place->flags); -	cur_flags |= place->flags & ~TTM_PL_MASK_CACHING; -  	mem->mem_type = place->mem_type; -	mem->placement = cur_flags; +	mem->placement = place->flags;  	spin_lock(&ttm_bo_glob.lru_lock);  	ttm_bo_del_from_lru(bo); @@ -922,7 +866,7 @@ static int ttm_bo_mem_placement(struct ttm_buffer_object *bo,  	return 0;  } -/** +/*   * Creates space for memory region @mem according to its type.   *   * This function first searches for free space in compatible memory types in @@ -947,7 +891,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,  		const struct ttm_place *place = &placement->placement[i];  		struct ttm_resource_manager *man; -		ret = ttm_bo_mem_placement(bo, place, mem, ctx); +		ret = ttm_bo_mem_placement(bo, place, mem);  		if (ret)  			continue; @@ -973,7 +917,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,  	for (i = 0; i < placement->num_busy_placement; ++i) {  		const struct ttm_place *place = &placement->busy_placement[i]; -		ret = ttm_bo_mem_placement(bo, place, mem, ctx); +		ret = ttm_bo_mem_placement(bo, place, mem);  		if (ret)  			continue; @@ -1001,15 +945,45 @@ error:  }  EXPORT_SYMBOL(ttm_bo_mem_space); +static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, +				     struct ttm_resource *mem, +				     struct ttm_operation_ctx *ctx, +				     struct ttm_place *hop) +{ +	struct ttm_placement hop_placement; +	int ret; +	struct ttm_resource hop_mem = *mem; + +	hop_mem.mm_node = NULL; +	hop_mem.mem_type = TTM_PL_SYSTEM; +	hop_mem.placement = 0; + +	hop_placement.num_placement = hop_placement.num_busy_placement = 1; +	hop_placement.placement = hop_placement.busy_placement = hop; + +	/* find space in the bounce domain */ +	ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx); +	if (ret) +		return ret; +	/* move to the bounce domain */ +	ret = ttm_bo_handle_move_mem(bo, &hop_mem, false, ctx, NULL); +	if (ret) +		return ret; +	return 0; +} +  static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,  			      struct ttm_placement *placement,  			      struct ttm_operation_ctx *ctx)  {  	int ret = 0; +	struct ttm_place hop;  	struct ttm_resource mem;  	dma_resv_assert_held(bo->base.resv); +	memset(&hop, 0, sizeof(hop)); +  	mem.num_pages = bo->num_pages;  	mem.size = mem.num_pages << PAGE_SHIFT;  	mem.page_alignment = bo->mem.page_alignment; @@ -1019,12 +993,25 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,  	/*  	 * Determine where to move the buffer. +	 * +	 * If driver determines move is going to need +	 * an extra step then it will return -EMULTIHOP +	 * and the buffer will be moved to the temporary +	 * stop and the driver will be called to make +	 * the second hop.  	 */ +bounce:  	ret = ttm_bo_mem_space(bo, placement, &mem, ctx);  	if (ret) -		goto out_unlock; -	ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx); -out_unlock: +		return ret; +	ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx, &hop); +	if (ret == -EMULTIHOP) { +		ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop); +		if (ret) +			return ret; +		/* try and move to final place now. */ +		goto bounce; +	}  	if (ret)  		ttm_resource_free(bo, &mem);  	return ret; @@ -1045,8 +1032,7 @@ static bool ttm_bo_places_compat(const struct ttm_place *places,  			continue;  		*new_flags = heap->flags; -		if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && -		    (mem->mem_type == heap->mem_type) && +		if ((mem->mem_type == heap->mem_type) &&  		    (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||  		     (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))  			return true; @@ -1100,9 +1086,6 @@ int ttm_bo_validate(struct ttm_buffer_object *bo,  		ret = ttm_bo_move_buffer(bo, placement, ctx);  		if (ret)  			return ret; -	} else { -		bo->mem.placement &= TTM_PL_MASK_CACHING; -		bo->mem.placement |= new_flags & ~TTM_PL_MASK_CACHING;  	}  	/*  	 * We might need to add a TTM. @@ -1170,8 +1153,9 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev,  	bo->mem.bus.offset = 0;  	bo->mem.bus.addr = NULL;  	bo->moving = NULL; -	bo->mem.placement = TTM_PL_FLAG_CACHED; +	bo->mem.placement = 0;  	bo->acc_size = acc_size; +	bo->pin_count = 0;  	bo->sg = sg;  	if (resv) {  		bo->base.resv = resv; @@ -1251,19 +1235,6 @@ int ttm_bo_init(struct ttm_bo_device *bdev,  }  EXPORT_SYMBOL(ttm_bo_init); -static size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, -			      unsigned long bo_size, -			      unsigned struct_size) -{ -	unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; -	size_t size = 0; - -	size += ttm_round_pot(struct_size); -	size += ttm_round_pot(npages * sizeof(void *)); -	size += ttm_round_pot(sizeof(struct ttm_tt)); -	return size; -} -  size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,  			   unsigned long bo_size,  			   unsigned struct_size) @@ -1273,56 +1244,11 @@ size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,  	size += ttm_round_pot(struct_size);  	size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); -	size += ttm_round_pot(sizeof(struct ttm_dma_tt)); +	size += ttm_round_pot(sizeof(struct ttm_tt));  	return size;  }  EXPORT_SYMBOL(ttm_bo_dma_acc_size); -int ttm_bo_create(struct ttm_bo_device *bdev, -			unsigned long size, -			enum ttm_bo_type type, -			struct ttm_placement *placement, -			uint32_t page_alignment, -			bool interruptible, -			struct ttm_buffer_object **p_bo) -{ -	struct ttm_buffer_object *bo; -	size_t acc_size; -	int ret; - -	bo = kzalloc(sizeof(*bo), GFP_KERNEL); -	if (unlikely(bo == NULL)) -		return -ENOMEM; - -	acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); -	ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, -			  interruptible, acc_size, -			  NULL, NULL, NULL); -	if (likely(ret == 0)) -		*p_bo = bo; - -	return ret; -} -EXPORT_SYMBOL(ttm_bo_create); - -int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) -{ -	struct ttm_resource_manager *man = ttm_manager_type(bdev, mem_type); - -	if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { -		pr_err("Illegal memory manager memory type %u\n", mem_type); -		return -EINVAL; -	} - -	if (!man) { -		pr_err("Memory type %u has not been initialized\n", mem_type); -		return 0; -	} - -	return ttm_resource_manager_force_list_clean(bdev, man); -} -EXPORT_SYMBOL(ttm_bo_evict_mm); -  static void ttm_bo_global_kobj_release(struct kobject *kobj)  {  	struct ttm_bo_global *glob = @@ -1409,6 +1335,8 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev)  			pr_debug("Swap list %d was clean\n", i);  	spin_unlock(&glob->lru_lock); +	ttm_pool_fini(&bdev->pool); +  	if (!ret)  		ttm_bo_global_release(); @@ -1433,9 +1361,10 @@ static void ttm_bo_init_sysman(struct ttm_bo_device *bdev)  int ttm_bo_device_init(struct ttm_bo_device *bdev,  		       struct ttm_bo_driver *driver, +		       struct device *dev,  		       struct address_space *mapping,  		       struct drm_vma_offset_manager *vma_manager, -		       bool need_dma32) +		       bool use_dma_alloc, bool use_dma32)  {  	struct ttm_bo_global *glob = &ttm_bo_glob;  	int ret; @@ -1450,12 +1379,12 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,  	bdev->driver = driver;  	ttm_bo_init_sysman(bdev); +	ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32);  	bdev->vma_manager = vma_manager;  	INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);  	INIT_LIST_HEAD(&bdev->ddestroy);  	bdev->dev_mapping = mapping; -	bdev->need_dma32 = need_dma32;  	mutex_lock(&ttm_global_mutex);  	list_add_tail(&bdev->device_list, &glob->device_list);  	mutex_unlock(&ttm_global_mutex); @@ -1502,12 +1431,13 @@ int ttm_bo_wait(struct ttm_buffer_object *bo,  }  EXPORT_SYMBOL(ttm_bo_wait); -/** +/*   * A buffer object shrink method that tries to swap out the first   * buffer object on the bo_global::swap_lru list.   */ -int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) +int ttm_bo_swapout(struct ttm_operation_ctx *ctx)  { +	struct ttm_bo_global *glob = &ttm_bo_glob;  	struct ttm_buffer_object *bo;  	int ret = -EBUSY;  	bool locked; @@ -1551,19 +1481,23 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)  	 * Move to system cached  	 */ -	if (bo->mem.mem_type != TTM_PL_SYSTEM || -	    bo->ttm->caching_state != tt_cached) { +	if (bo->mem.mem_type != TTM_PL_SYSTEM) {  		struct ttm_operation_ctx ctx = { false, false };  		struct ttm_resource evict_mem; +		struct ttm_place hop; + +		memset(&hop, 0, sizeof(hop));  		evict_mem = bo->mem;  		evict_mem.mm_node = NULL; -		evict_mem.placement = TTM_PL_FLAG_CACHED; +		evict_mem.placement = 0;  		evict_mem.mem_type = TTM_PL_SYSTEM; -		ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); -		if (unlikely(ret != 0)) +		ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx, &hop); +		if (unlikely(ret != 0)) { +			WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n");  			goto out; +		}  	}  	/** @@ -1584,7 +1518,7 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)  	if (bo->bdev->driver->swap_notify)  		bo->bdev->driver->swap_notify(bo); -	ret = ttm_tt_swapout(bo->bdev, bo->ttm, bo->persistent_swap_storage); +	ret = ttm_tt_swapout(bo->bdev, bo->ttm);  out:  	/** @@ -1599,17 +1533,6 @@ out:  }  EXPORT_SYMBOL(ttm_bo_swapout); -void ttm_bo_swapout_all(void) -{ -	struct ttm_operation_ctx ctx = { -		.interruptible = false, -		.no_wait_gpu = false -	}; - -	while (ttm_bo_swapout(&ttm_bo_glob, &ctx) == 0); -} -EXPORT_SYMBOL(ttm_bo_swapout_all); -  void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)  {  	if (bo->ttm == NULL) @@ -1619,12 +1542,3 @@ void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)  	bo->ttm = NULL;  } -int ttm_bo_tt_bind(struct ttm_buffer_object *bo, struct ttm_resource *mem) -{ -	return bo->bdev->driver->ttm_tt_bind(bo->bdev, bo->ttm, mem); -} - -void ttm_bo_tt_unbind(struct ttm_buffer_object *bo) -{ -	bo->bdev->driver->ttm_tt_unbind(bo->bdev, bo->ttm); -} | 
